diff --git a/alloy-modules/kubernetes/logs/annotations-scrape.alloy b/alloy-modules/kubernetes/logs/annotations-scrape.alloy index 66bbceb1..6fe80de1 100644 --- a/alloy-modules/kubernetes/logs/annotations-scrape.alloy +++ b/alloy-modules/kubernetes/logs/annotations-scrape.alloy @@ -3,7 +3,7 @@ Module Components: annotations_scrape Description: Scrapes targets for logs based on kubernetes Pod annotations Annotations: - logs.grafana.com/ingest: true + logs.grafana.com/scrape: true logs.grafana.com/tenant: "primary" */ @@ -22,11 +22,7 @@ declare "annotations_scrape" { default = ".*" } - // arguments for kubernetes discovery - argument "namespaces" { - comment = "The namespaces to look for targets in (default: [\"kube-system\"] is all namespaces)" - optional = true - } + argument "cluster" { } argument "annotation_prefix" { comment = "The annotation_prefix to use (default: logs.grafana.com)" @@ -43,10 +39,6 @@ declare "annotations_scrape" { // find all pods discovery.kubernetes "annotation_logs" { role = "pod" - - namespaces { - names = coalesce(argument.namespaces.value, []) - } } // filter logs by kubernetes annotations @@ -54,7 +46,7 @@ declare "annotations_scrape" { targets = discovery.kubernetes.annotation_logs.targets // allow pods to declare their logs to be ingested or not, the default is true - // i.e. logs.grafana.com/ingest: false + // i.e. logs.grafana.com/scrape: false rule { action = "keep" source_labels = [ @@ -80,6 +72,13 @@ declare "annotations_scrape" { target_label = "instance" } + // set the cluster label + rule { + action = "replace" + replacement = argument.cluster.value + target_label = "cluster" + } + // set the namespace label rule { source_labels = ["__meta_kubernetes_namespace"] diff --git a/alloy-modules/kubernetes/logs/k8s-events.alloy b/alloy-modules/kubernetes/logs/k8s-events.alloy index 8c5f3a2d..af654f46 100644 --- a/alloy-modules/kubernetes/logs/k8s-events.alloy +++ b/alloy-modules/kubernetes/logs/k8s-events.alloy @@ -15,9 +15,7 @@ declare "kubernetes_cluster_events" { optional = true } - argument "cluster" { - optional = true - } + argument "cluster" { } loki.source.kubernetes_events "cluster_events" { job_name = coalesce(argument.job_label.value, "integrations/kubernetes/eventhandler") @@ -28,7 +26,7 @@ declare "kubernetes_cluster_events" { loki.process "logs_service" { stage.static_labels { values = { - cluster = coalesce(argument.cluster.value, "k3d"), + cluster = argument.cluster.value, } } forward_to = argument.forward_to.value diff --git a/alloy-modules/kubernetes/logs/rules-to-loki.alloy b/alloy-modules/kubernetes/logs/rules-to-loki.alloy new file mode 100644 index 00000000..aa0a6194 --- /dev/null +++ b/alloy-modules/kubernetes/logs/rules-to-loki.alloy @@ -0,0 +1,40 @@ +/* +Module Components: rules_to_loki +Description: Auto discovers PrometheusRule Kubernetes resources and loads them into a Loki instance. +*/ + +declare "rules_to_loki" { + + /***************************************************************** + * ARGUMENTS + *****************************************************************/ + argument "address" { + comment = "URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)" + optional = true + } + + argument "tenant" { + comment = "Mimir tenant ID. (default: fake)" + optional = true + } + + /******************************************** + * Kubernetes Prometheus Rules To Loki + ********************************************/ + loki.rules.kubernetes "rules_to_loki" { + address = coalesce(argument.address.value, "http://nginx.gateway.svc:3100") + tenant_id = coalesce(argument.tenant.value, "anonymous") + + // rule_namespace_selector { + // match_labels = { + // auto_rules_to_loki= "true", + // } + // } + + rule_selector { + match_labels = { + auto_rules_to_loki = "true", + } + } + } +} diff --git a/alloy-modules/kubernetes/metrics/annotations-scrape.alloy b/alloy-modules/kubernetes/metrics/annotations-scrape.alloy new file mode 100644 index 00000000..2ee4a74f --- /dev/null +++ b/alloy-modules/kubernetes/metrics/annotations-scrape.alloy @@ -0,0 +1,606 @@ +/* +Module Components: annotations_scrape +Description: Scrapes targets for metrics based on kubernetes annotations + +Note: Every argument except for "forward_to" is optional, and does have a defined default value. However, the values for these + arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... "). + This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will + does not override the value passed in, where coalesce() will return the first non-null value. + +Kubernetes Annotation Auto-Scraping +------------------------------------------------------------------------------------------------------------------------------------ +This module is meant to be used to automatically scrape targets based on a certain role and set of annotations. This module can be consumed +multiple times with different roles. The supported roles are: + + - pod + - service + - endpoints + +Typically, if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality you would use role="endpoints", if +mimicking the behavior of the PodMonitor functionality you would use role="pod". It is important to understand that with endpoints, +the target is typically going to be a pod, and whatever annotations that are set on the service will automatically be propagated to the +endpoints. This is why the role "endpoints" is used, because it will scrape the pod, but also consider the service annotations. Using +role="endpoints", which scrape each endpoint associated to the service. If role="service" is used, it will only scrape the service, only +hitting one of the endpoints associated to the service. + +This is where you must consider your scraping strategy, for example if you scrape a service like "kube-state-metrics" using +role="endpoints" you should only have a single replica of the kube-state-metrics pod, if you have multiple replicas, you should use +role="service" or a separate non-annotation job completely. Scraping a service instead of endpoints, is typically a rare use case, but +it is supported. + +There are other considerations for using annotation based scraping as well, which is metric relabeling rules that happen post scrape. If +you have a target that you want to apply a bunch of relabelings to or a very large metrics response payload, performance wise it will be +better to have a separate job for that target, rather than using use annotations. As every targert will go through the ssame relabeling. +Typical deployment strategies/options would be: + +Option #1 (recommended): + - Annotation Scraping for role="endpoints" + - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, etc.) or large metric payloads + - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, kube-scheduler, etc.) + +Option #2: + - Annotation Scraping for role="pod" + - Annotation Scraping for role="service" (i.e. kube-state-metrics, node-exporter, etc.) + - Separate Jobs for specific use cases or large metric payloads + - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, kube-scheduler, etc.) + +At no point should you use role="endpoints" and role="pod" together, as this will result in duplicate targets being scraped, thus +generating duplicate metrics. If you want to scrape both the pod and the service, use Option #2. + +Each port attached to an service/pod/endpoint is an eligible target, oftentimes it will have multiple ports. +There may be instances when you want to scrape all ports or some ports and not others. To support this +the following annotations are available: + + metrics.grafana.com/scrape: true + +the default scraping scheme is http, this can be specified as a single value which would override, the schema being used for all +ports attached to the target: + + metrics.grafana.com/scheme: https + +the default path to scrape is /metrics, this can be specified as a single value which would override, the scrape path being used +for all ports attached to the target: + + metrics.grafana.com/path: /metrics/some_path + +the default port to scrape is the target port, this can be specified as a single value which would override the scrape port being +used for all ports attached to the target, note that even if aan target had multiple targets, the relabel_config targets are +deduped before scraping: + + metrics.grafana.com/port: 8080 + +the default interval to scrape is 1m, this can be specified as a single value which would override, the scrape interval being used +for all ports attached to the target: + + metrics.grafana.com/interval: 5m + +the default timeout for scraping is 10s, this can be specified as a single value which would override, the scrape interval being +used for all ports attached to the target: + + metrics.grafana.com/timeout: 30s + +the default job is namespace/{{ service name }} or namespace/{{ controller_name }} depending on the role, there may be instances +in which a different job name is required because of a set of dashboards, rules, etc. to support this there is a job annotation +which will override the default value: + + metrics.grafana.com/job: integrations/kubernetes/kube-state-metrics +*/ + +declare "annotations_scrape" { + + /***************************************************************** + * ARGUMENTS + *****************************************************************/ + argument "forward_to" { + comment = "Must be a list(MetricssReceiver) where collected metrics should be forwarded to" + } + + argument "tenant" { + comment = "The tenant to filter metrics to. This does not have to be the tenantId, this is the value to look for in the metrics.agent.grafana.com/tenant annotation, and this can be a regex." + optional = true + } + + argument "cluster" { } + + argument "annotation_prefix" { + comment = "The annotation_prefix to use (default: metrics.grafana.com)" + default = "metrics.grafana.com" + optional = true + } + + argument "role" { + comment = "The role to use when looking for targets to scrape via annotations, can be: endpoints, service, pod (default: endpoints)" + optional = true + } + + argument "__sd_annotation" { + optional = true + comment = "The logic is used to transform the annotation argument into a valid label name by removing unsupported characters." + default = replace(replace(replace(coalesce(argument.annotation_prefix.value, "metrics.grafana.com"), ".", "_"), "/", "_"), "-", "_") + } + + argument "__pod_role" { + comment = "Most annotation targets service or pod that is all you want, however if the role is endpoints you want the pod" + optional = true + default = replace(coalesce(argument.role.value, "endpoints"), "endpoints", "pod") + } + + argument "__service_role" { + comment = "Most annotation targets service or pod that is all you want, however if the role is endpoints you we also want to consider service annotations" + optional = true + default = replace(coalesce(argument.role.value, "endpoints"), "endpoints", "service") + } + + argument "scrape_port_named_metrics" { + comment = "Whether or not to automatically scrape endpoints that have a port with 'metrics' in the name" + optional = true + default = false + } + + argument "keep_metrics" { + comment = "A regex of metrics to keep (default: (.+))" + optional = true + } + + argument "drop_metrics" { + comment = "A regex of metrics to drop (default: \"\")" + optional = true + } + + argument "scrape_interval" { + comment = "How often to scrape metrics from the targets (default: 60s)" + optional = true + } + + argument "scrape_timeout" { + comment = "How long before a scrape times out (default: 10s)" + optional = true + } + + /***************************************************************** + * Targets From Docker Discovery + *****************************************************************/ + discovery.kubernetes "annotation_metrics" { + role = coalesce(argument.role.value, "endpoints") + + selectors { + role = coalesce(argument.role.value, "endpoints") + } + } + + /***************************************************************** + * Discovery Relabelings (pre-scrape) + *****************************************************************/ + // filter metrics by kubernetes annotations + discovery.relabel "annotation_metrics_filter" { + targets = discovery.kubernetes.annotation_metrics.targets + + /**************************************************************************************************************** + * Handle Targets to Keep or Drop + ****************************************************************************************************************/ + // allow resources to declare their metrics scraped or not + // Example Annotation: + // metrics.grafana.com/scrape: false + // + // the label prometheus.io/service-monitor: "false" is a common label for headless services, when performing endpoint + // service discovery, if there is both a load-balanced service and headless service, this can result in duplicate + // scrapes if the name of the service is attached as a label. any targets with this label or annotation set should be dropped + rule { + action = "replace" + replacement = "false" + target_label = "__tmp_scrape" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_scrape", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_scrape", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_prometheus_io_service_monitor", + ] + separator = ";" + // only allow empty or true, otherwise defaults to false + regex = "^(?:;*)?(true)(;|true)*$" + replacement = "$1" + target_label = "__tmp_scrape" + } + + // add a __tmp_scrape_port_named_metrics from the argument.scrape_port_named_metrics + rule { + replacement = format("%t", argument.scrape_port_named_metrics.value) + target_label = "__tmp_scrape_port_named_metrics" + } + + // only keep targets that have scrape: true or "metrics" in the port name if the argument scrape_port_named_metrics + rule { + action = "keep" + source_labels = [ + "__tmp_scrape", + "__tmp_scrape_port_named_metrics", + // endpoints is the role and most meta labels started with "endpoints", however the port name is an exception and starts with "endpoint" + "__meta_kubernetes_" + replace(coalesce(argument.role.value, "endpoints"), "endpoints", "endpoint") + "_port_name", + ] + separator = ";" + regex = "^(true;.*|(|true);true;(.*metrics.*))$" + } + + // only keep targets where the pod is running or the pod_phase is empty and is not an init container. This will only exist for role="pod" or + // potentially role="endpoints", if it is a service the value is empty and thus allowed to pass, if it is an endpoint but not associated to a + // pod but rather a static IP or hostname, that could be outside of kubernetes allow endpoints to declare what tenant their metrics should be + // written to + rule { + action = "keep" + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "^(?i)(Running|)$" + } + + rule { + action = "keep" + source_labels = ["__meta_kubernetes_pod_ready"] + regex = "^(true|)$" + } + // if the container is an init container, drop it + rule { + action = "drop" + source_labels = ["__meta_kubernetes_pod_container_init"] + regex = "^(true)$" + } + + // allow resources to declare their metrics the tenant their metrics should be sent to, + // Example Annotation: + // metrics.grafana.com/tenant: primary + // + // Note: This does not necessarily have to be the actual tenantId, it can be a friendly name as well that is simply used + // to determine if the metrics should be gathered for the current tenant + rule { + action = "keep" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_tenant", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_tenant", + ] + regex = "^(" + coalesce(argument.tenant.value, ".*") + ")$" + } + + /**************************************************************************************************************** + * Handle Setting Scrape Metadata i.e. path, port, interval etc. + ****************************************************************************************************************/ + // allow resources to declare the protocol to use when collecting metrics, the default value is "http", + // Example Annotation: + // metrics.grafana.com/scheme: http + rule { + action = "replace" + replacement = "http" + target_label = "__scheme__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_scheme", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_scheme", + ] + separator = ";" + regex = "^(?:;*)?(https?).*$" + replacement = "$1" + target_label = "__scheme__" + } + + // allow resources to declare the port to use when collecting metrics, the default value is the discovered port from + // Example Annotation: + // metrics.grafana.com/port: 9090 + rule { + action = "replace" + source_labels = [ + "__address__", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_port", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_port", + ] + separator = ";" + regex = "^([^:]+)(?::\\d+)?;(\\d+)$" + replacement = "$1:$2" + target_label = "__address__" + } + + // allow resources to declare their the path to use when collecting their metrics, the default value is "/metrics", + // Example Annotation: + // metrics.grafana.com/path: /metrics/foo + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_path", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__metrics_path__" + } + + // allow resources to declare how often their metrics should be collected, the default value is 1m, + // the following duration formats are supported (s|m|ms|h|d): + // Example Annotation: + // metrics.grafana.com/interval: 5m + rule { + action = "replace" + replacement = coalesce(argument.scrape_interval.value, "60s") + target_label = "__scrape_interval__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_interval", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_interval", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_interval__" + } + + // allow resources to declare the timeout of the scrape request, the default value is 10s, + // the following duration formats are supported (s|m|ms|h|d): + // Example Annotation: + // metrics.grafana.com/timeout: 30s + rule { + action = "replace" + replacement = coalesce(argument.scrape_timeout.value, "10s") + target_label = "__scrape_timeout__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_timeout", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_timeout", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_timeout__" + } + + /**************************************************************************************************************** + * Handle Setting Common Labels + ****************************************************************************************************************/ + // set a source label + rule { + action = "replace" + replacement = "kubernetes" + target_label = "source" + } + + // set the cluster label + rule { + action = "replace" + replacement = argument.cluster.value + target_label = "cluster" + } + + // set the namespace label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + // set the target name label i.e. service name, pod name, etc. + // if the role is endpoints, the first valued field is used which would be __meta_kubernetes_pod_name, if the pod name is empty + // then the endpoint name would be used + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.__pod_role.value + "_name", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_name", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = argument.__pod_role.value + } + + // set a default job label to be the namespace/pod_controller_name or namespace/service_name + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_namespace", + "__meta_kubernetes_pod_controller_name", + argument.__pod_role.value, + ] + separator = ";" + regex = "^([^;]+)(?:;*)?([^;]+).*$" + replacement = "$1/$2" + target_label = "job" + } + + // if the controller is a ReplicaSet, drop the hash from the end of the ReplicaSet + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_pod_controller_type", + "__meta_kubernetes_namespace", + "__meta_kubernetes_pod_controller_name", + ] + separator = ";" + regex = "^(?:ReplicaSet);([^;]+);([^;]+)-.+$" + replacement = "$1/$2" + target_label = "job" + } + + // allow resources to declare their the job label value to use when collecting their metrics, the default value is "", + // Example Annotation: + // metrics.grafana.com/job: integrations/kubernetes/cadvisor + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_job", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_job", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "job" + } + + // set the app name if specified as metadata labels "app:" or "app.kubernetes.io/name:" + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_app_kubernetes_io_name", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_k8s_app", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_app", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_app_kubernetes_io_name", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_k8s_app", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_app", + "__meta_kubernetes_" + argument.__service_role.value + "_label_app_kubernetes_io_name", + "__meta_kubernetes_" + argument.__service_role.value + "_label_k8s_app", + "__meta_kubernetes_" + argument.__service_role.value + "_label_app", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "app" + } + + // set the app component if specified as metadata labels "component:" or "app.kubernetes.io/component:" + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_app_kubernetes_io_component", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_component", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_app_kubernetes_io_component", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_component", + "__meta_kubernetes_" + argument.__service_role.value + "_label_app_kubernetes_io_component", + "__meta_kubernetes_" + argument.__service_role.value + "_label_component", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "component" + } + + // set the version if specified as metadata labels "version:" or "app.kubernetes.io/version:" or "app_version:" + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_app_kubernetes_io_version", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_version", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_app_kubernetes_io_version", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_version", + "__meta_kubernetes_" + argument.__service_role.value + "_label_app_kubernetes_io_version", + "__meta_kubernetes_" + argument.__service_role.value + "_label_version", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "version" + } + + // set a workload label if the resource is a pod + // example: grafana-agent-68nv9 becomes DaemonSet/grafana-agent + rule { + source_labels = [ + "__meta_kubernetes_pod_controller_kind", + "__meta_kubernetes_pod_controller_name", + ] + separator = ";" + regex = "(.+);(.+)" + replacement = "$1/$2" + target_label = "workload" + } + // remove the hash from the ReplicaSet + rule { + source_labels = ["workload"] + regex = "(ReplicaSet/.+)-.+" + target_label = "workload" + } + } + + // only keep http targets + discovery.relabel "http_annotations" { + targets = discovery.relabel.annotation_metrics_filter.output + + rule { + action = "keep" + source_labels = ["__scheme__"] + regex = "http" + } + } + + // only keep https targets + discovery.relabel "https_annotations" { + targets = discovery.relabel.annotation_metrics_filter.output + + rule { + action = "keep" + source_labels = ["__scheme__"] + regex = "https" + } + } + + /***************************************************************** + * Prometheus Scrape Labels Targets + *****************************************************************/ + // scrape http only targtetsa + prometheus.scrape "http_annotations" { + targets = discovery.relabel.http_annotations.output + + job_name = "annotation-metrics-http" + scheme = "http" + scrape_interval = coalesce(argument.scrape_interval.value, "60s") + scrape_timeout = coalesce(argument.scrape_timeout.value, "10s") + + enable_protobuf_negotiation = true + scrape_classic_histograms = true + + clustering { + enabled = true + } + + forward_to = [prometheus.relabel.annotations.receiver] + } + + // scrape https only targtets + prometheus.scrape "https_annotations" { + targets = discovery.relabel.https_annotations.output + + job_name = "annotation-metrics-https" + scheme = "https" + scrape_interval = coalesce(argument.scrape_interval.value, "60s") + scrape_timeout = coalesce(argument.scrape_timeout.value, "10s") + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + tls_config { + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + insecure_skip_verify = false + server_name = "kubernetes" + } + + enable_protobuf_negotiation = true + scrape_classic_histograms = true + + clustering { + enabled = true + } + + forward_to = [prometheus.relabel.annotations.receiver] + } + + /***************************************************************** + * Prometheus Metric Relabelings (post-scrape) + *****************************************************************/ + // perform generic relabeling using keep_metrics and drop_metrics + prometheus.relabel "annotations" { + forward_to = argument.forward_to.value + // keep only metrics that match the keep_metrics regex + rule { + action = "keep" + source_labels = ["__name__"] + regex = coalesce(argument.keep_metrics.value, "(.+)") + } + + // drop metrics that match the drop_metrics regex + rule { + action = "drop" + source_labels = ["__name__"] + regex = coalesce(argument.drop_metrics.value, "") + } + } +} diff --git a/alloy-modules/kubernetes/metrics/podmonitors-scrape.alloy b/alloy-modules/kubernetes/metrics/podmonitors-scrape.alloy index 540e2ff0..100eff2a 100644 --- a/alloy-modules/kubernetes/metrics/podmonitors-scrape.alloy +++ b/alloy-modules/kubernetes/metrics/podmonitors-scrape.alloy @@ -13,6 +13,18 @@ declare "podmonitors_scrape" { optional = false } + argument "cluster" { } + + argument "keep_metrics" { + comment = "A regex of metrics to keep (default: (.+))" + optional = true + } + + argument "drop_metrics" { + comment = "A regex of metrics to drop (default: \"\")" + optional = true + } + argument "scrape_interval" { comment = "How often to scrape metrics from the targets (default: 60s)" optional = true @@ -27,7 +39,7 @@ declare "podmonitors_scrape" { * Kubernetes Auto Scrape PodMonitors *****************************************************************/ prometheus.operator.podmonitors "scrape" { - forward_to = argument.forward_to.value + forward_to = [prometheus.relabel.podmonitors.receiver] scrape { default_scrape_interval = coalesce(argument.scrape_interval.value, "60s") @@ -46,4 +58,33 @@ declare "podmonitors_scrape" { // } // } } + + /***************************************************************** + * Prometheus Metric Relabelings (post-scrape) + *****************************************************************/ + // perform generic relabeling using keep_metrics and drop_metrics + prometheus.relabel "podmonitors" { + forward_to = argument.forward_to.value + + // set the cluster label + rule { + action = "replace" + replacement = argument.cluster.value + target_label = "cluster" + } + + // keep only metrics that match the keep_metrics regex + rule { + action = "keep" + source_labels = ["__name__"] + regex = coalesce(argument.keep_metrics.value, "(.+)") + } + + // drop metrics that match the drop_metrics regex + rule { + action = "drop" + source_labels = ["__name__"] + regex = coalesce(argument.drop_metrics.value, "") + } + } } diff --git a/alloy-modules/kubernetes/metrics/rules-to-mimir.alloy b/alloy-modules/kubernetes/metrics/rules-to-mimir.alloy new file mode 100644 index 00000000..3dbd975a --- /dev/null +++ b/alloy-modules/kubernetes/metrics/rules-to-mimir.alloy @@ -0,0 +1,40 @@ +/* +Module Components: rules_to_mimir +Description: Auto discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance. +*/ + +declare "rules_to_mimir" { + + /***************************************************************** + * ARGUMENTS + *****************************************************************/ + argument "address" { + comment = "URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)" + optional = true + } + + argument "tenant" { + comment = "Mimir tenant ID. (default: anonymous)" + optional = true + } + + /******************************************** + * Kubernetes Prometheus Rules To Mimir + ********************************************/ + mimir.rules.kubernetes "rules_to_mimir" { + address = coalesce(argument.address.value, "http://nginx.gateway.svc:8080") + tenant_id = coalesce(argument.tenant.value, "anonymous") + + // rule_namespace_selector { + // match_labels = { + // auto_rules_to_mimir= "true", + // } + // } + + // rule_selector { + // match_labels = { + // auto_rules_to_mimir= "true", + // } + // } + } +} diff --git a/alloy-modules/kubernetes/metrics/servicemonitors-scrape.alloy b/alloy-modules/kubernetes/metrics/servicemonitors-scrape.alloy index d221e276..319e1423 100644 --- a/alloy-modules/kubernetes/metrics/servicemonitors-scrape.alloy +++ b/alloy-modules/kubernetes/metrics/servicemonitors-scrape.alloy @@ -13,6 +13,18 @@ declare "servicemonitors_scrape" { optional = false } + argument "cluster" { } + + argument "keep_metrics" { + comment = "A regex of metrics to keep (default: (.+))" + optional = true + } + + argument "drop_metrics" { + comment = "A regex of metrics to drop (default: \"\")" + optional = true + } + argument "scrape_interval" { comment = "How often to scrape metrics from the targets (default: 60s)" optional = true @@ -27,7 +39,7 @@ declare "servicemonitors_scrape" { * Kubernetes Auto Scrape ServiceMonitors *****************************************************************/ prometheus.operator.servicemonitors "scrape" { - forward_to = argument.forward_to.value + forward_to = [prometheus.relabel.servicemonitors.receiver] scrape { default_scrape_interval = coalesce(argument.scrape_interval.value, "60s") @@ -38,4 +50,33 @@ declare "servicemonitors_scrape" { enabled = true } } + + /***************************************************************** + * Prometheus Metric Relabelings (post-scrape) + *****************************************************************/ + // perform generic relabeling using keep_metrics and drop_metrics + prometheus.relabel "servicemonitors" { + forward_to = argument.forward_to.value + + // set the cluster label + rule { + action = "replace" + replacement = argument.cluster.value + target_label = "cluster" + } + + // keep only metrics that match the keep_metrics regex + rule { + action = "keep" + source_labels = ["__name__"] + regex = coalesce(argument.keep_metrics.value, "(.+)") + } + + // drop metrics that match the drop_metrics regex + rule { + action = "drop" + source_labels = ["__name__"] + regex = coalesce(argument.drop_metrics.value, "") + } + } } diff --git a/alloy-modules/kubernetes/profiles/annotations-scrape.alloy b/alloy-modules/kubernetes/profiles/annotations-scrape.alloy index 33d82ce5..05154c7d 100644 --- a/alloy-modules/kubernetes/profiles/annotations-scrape.alloy +++ b/alloy-modules/kubernetes/profiles/annotations-scrape.alloy @@ -13,6 +13,8 @@ declare "annotations_scrape" { comment = "Must be a list(ProfilessReceiver) where collected logs should be forwarded to" } + argument "cluster" { } + discovery.kubernetes "pyroscope_kubernetes" { role = "pod" } @@ -47,6 +49,13 @@ declare "annotations_scrape" { regex = "__meta_kubernetes_pod_label_(.+)" } + // set the cluster label + rule { + action = "replace" + replacement = argument.cluster.value + target_label = "cluster" + } + rule { action = "replace" source_labels = ["__meta_kubernetes_namespace"] diff --git a/alloy-modules/kubernetes/traces/process-and-transform.alloy b/alloy-modules/kubernetes/traces/process-and-transform.alloy index 7ab57c49..15d206f8 100644 --- a/alloy-modules/kubernetes/traces/process-and-transform.alloy +++ b/alloy-modules/kubernetes/traces/process-and-transform.alloy @@ -14,6 +14,8 @@ declare "process_and_transform" { comment = "Must be a list(TracesReceiver) where collected traces should be forwarded to" } + argument "cluster" { } + argument "logs_forward_to" { comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" } @@ -22,11 +24,6 @@ declare "process_and_transform" { comment = "Must be a list(MetricsReceiver) where collected metrics should be forwarded to" } - argument "cluster" { - optional = true - default = "k3d-k3s-codelab" - } - argument "otlp_http_endpoint" { optional = true default = "0.0.0.0:4318" @@ -60,9 +57,7 @@ declare "process_and_transform" { } output { - metrics = [otelcol.processor.batch.default.input] - logs = [otelcol.processor.resourcedetection.default.input] - traces = [otelcol.processor.resourcedetection.default.input] + traces = [otelcol.processor.k8sattributes.default.input] } } @@ -70,6 +65,10 @@ declare "process_and_transform" { * Otelcol for Metrics Logs Traces *****************************************************************/ otelcol.receiver.otlp "default" { + debug_metrics { + disable_high_cardinality_metrics = true + } + grpc { endpoint = argument.otlp_grpc_endpoint.value } @@ -79,7 +78,7 @@ declare "process_and_transform" { } output { - metrics = [otelcol.processor.batch.default.input] + metrics = [otelcol.processor.resourcedetection.default.input] logs = [otelcol.processor.resourcedetection.default.input] traces = [ otelcol.processor.resourcedetection.default.input, @@ -89,14 +88,36 @@ declare "process_and_transform" { } otelcol.processor.resourcedetection "default" { - detectors = ["env"] + detectors = ["env", "system"] + + system { + hostname_sources = ["os"] + } output { + metrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input] + logs = [otelcol.processor.k8sattributes.default.input] traces = [otelcol.processor.k8sattributes.default.input] } } + otelcol.processor.transform "add_metric_datapoint_attributes" { + error_mode = "ignore" + + metric_statements { + context = "datapoint" + statements = [ + "set(attributes[\"deployment.environment\"], resource.attributes[\"deployment.environment\"])", + "set(attributes[\"service.version\"], resource.attributes[\"service.version\"])", + ] + } + + output { + metrics = [otelcol.processor.k8sattributes.default.input] + } + } + otelcol.processor.k8sattributes "default" { extract { metadata = [ @@ -120,14 +141,44 @@ declare "process_and_transform" { } output { - logs = [otelcol.processor.transform.add_resource_attributes.input] - traces = [otelcol.processor.transform.add_resource_attributes.input] + metrics = [otelcol.processor.transform.default.input] + logs = [otelcol.processor.transform.default.input] + traces = [ + otelcol.processor.transform.default.input, + otelcol.connector.host_info.default.input, + ] } } - otelcol.processor.transform "add_resource_attributes" { + otelcol.connector.host_info "default" { + host_identifiers = ["k8s.node.name"] + + output { + metrics = [otelcol.processor.batch.host_info_batch.input] + } + } + + otelcol.processor.batch "host_info_batch" { + output { + metrics = [otelcol.exporter.prometheus.host_info_metrics.input] + } + } + + otelcol.exporter.prometheus "host_info_metrics" { + add_metric_suffixes = false + forward_to = argument.metrics_forward_to.value + } + + otelcol.processor.transform "default" { error_mode = "ignore" + metric_statements { + context = "resource" + statements = [ + `set(attributes["k8s.cluster.name"], "k3d-k3s-codelab") where attributes["k8s.cluster.name"] == nil`, + ] + } + log_statements { context = "resource" statements = [ @@ -141,22 +192,42 @@ declare "process_and_transform" { trace_statements { context = "resource" statements = [ + `limit(attributes, 100, [])`, + `truncate_all(attributes, 4096)`, `set(attributes["k8s.cluster.name"], "k3d-k3s-codelab") where attributes["k8s.cluster.name"] == nil`, ] } + trace_statements { + context = "span" + statements = [ + `limit(attributes, 100, [])`, + `truncate_all(attributes, 4096)`, + ] + } + output { - logs = [otelcol.processor.filter.default.input] - traces = [otelcol.processor.filter.default.input] + metrics = [otelcol.processor.filter.default.input] + logs = [otelcol.processor.filter.default.input] + traces = [otelcol.processor.filter.default.input] } } otelcol.processor.filter "default" { error_mode = "ignore" + traces { + span = [ + "attributes[\"http.route\"] == \"/live\"", + "attributes[\"http.route\"] == \"/healthy\"", + "attributes[\"http.route\"] == \"/ready\"", + ] + } + output { - logs = [otelcol.processor.batch.default.input] - traces = [otelcol.processor.batch.default.input] + metrics = [otelcol.processor.batch.default.input] + logs = [otelcol.processor.batch.default.input] + traces = [otelcol.processor.batch.default.input] } } diff --git a/kubernetes/common/alloy/configs/kubernetes/logs/annotations-scrape.alloy b/kubernetes/common/alloy/configs/kubernetes/logs/annotations-scrape.alloy index 66bbceb1..6fe80de1 100644 --- a/kubernetes/common/alloy/configs/kubernetes/logs/annotations-scrape.alloy +++ b/kubernetes/common/alloy/configs/kubernetes/logs/annotations-scrape.alloy @@ -3,7 +3,7 @@ Module Components: annotations_scrape Description: Scrapes targets for logs based on kubernetes Pod annotations Annotations: - logs.grafana.com/ingest: true + logs.grafana.com/scrape: true logs.grafana.com/tenant: "primary" */ @@ -22,11 +22,7 @@ declare "annotations_scrape" { default = ".*" } - // arguments for kubernetes discovery - argument "namespaces" { - comment = "The namespaces to look for targets in (default: [\"kube-system\"] is all namespaces)" - optional = true - } + argument "cluster" { } argument "annotation_prefix" { comment = "The annotation_prefix to use (default: logs.grafana.com)" @@ -43,10 +39,6 @@ declare "annotations_scrape" { // find all pods discovery.kubernetes "annotation_logs" { role = "pod" - - namespaces { - names = coalesce(argument.namespaces.value, []) - } } // filter logs by kubernetes annotations @@ -54,7 +46,7 @@ declare "annotations_scrape" { targets = discovery.kubernetes.annotation_logs.targets // allow pods to declare their logs to be ingested or not, the default is true - // i.e. logs.grafana.com/ingest: false + // i.e. logs.grafana.com/scrape: false rule { action = "keep" source_labels = [ @@ -80,6 +72,13 @@ declare "annotations_scrape" { target_label = "instance" } + // set the cluster label + rule { + action = "replace" + replacement = argument.cluster.value + target_label = "cluster" + } + // set the namespace label rule { source_labels = ["__meta_kubernetes_namespace"] diff --git a/kubernetes/common/alloy/configs/kubernetes/logs/k8s-events.alloy b/kubernetes/common/alloy/configs/kubernetes/logs/k8s-events.alloy index 8c5f3a2d..af654f46 100644 --- a/kubernetes/common/alloy/configs/kubernetes/logs/k8s-events.alloy +++ b/kubernetes/common/alloy/configs/kubernetes/logs/k8s-events.alloy @@ -15,9 +15,7 @@ declare "kubernetes_cluster_events" { optional = true } - argument "cluster" { - optional = true - } + argument "cluster" { } loki.source.kubernetes_events "cluster_events" { job_name = coalesce(argument.job_label.value, "integrations/kubernetes/eventhandler") @@ -28,7 +26,7 @@ declare "kubernetes_cluster_events" { loki.process "logs_service" { stage.static_labels { values = { - cluster = coalesce(argument.cluster.value, "k3d"), + cluster = argument.cluster.value, } } forward_to = argument.forward_to.value diff --git a/kubernetes/common/alloy/configs/kubernetes/logs/rules-to-loki.alloy b/kubernetes/common/alloy/configs/kubernetes/logs/rules-to-loki.alloy new file mode 100644 index 00000000..aa0a6194 --- /dev/null +++ b/kubernetes/common/alloy/configs/kubernetes/logs/rules-to-loki.alloy @@ -0,0 +1,40 @@ +/* +Module Components: rules_to_loki +Description: Auto discovers PrometheusRule Kubernetes resources and loads them into a Loki instance. +*/ + +declare "rules_to_loki" { + + /***************************************************************** + * ARGUMENTS + *****************************************************************/ + argument "address" { + comment = "URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)" + optional = true + } + + argument "tenant" { + comment = "Mimir tenant ID. (default: fake)" + optional = true + } + + /******************************************** + * Kubernetes Prometheus Rules To Loki + ********************************************/ + loki.rules.kubernetes "rules_to_loki" { + address = coalesce(argument.address.value, "http://nginx.gateway.svc:3100") + tenant_id = coalesce(argument.tenant.value, "anonymous") + + // rule_namespace_selector { + // match_labels = { + // auto_rules_to_loki= "true", + // } + // } + + rule_selector { + match_labels = { + auto_rules_to_loki = "true", + } + } + } +} diff --git a/kubernetes/common/alloy/configs/kubernetes/metrics/annotations-scrape.alloy b/kubernetes/common/alloy/configs/kubernetes/metrics/annotations-scrape.alloy new file mode 100644 index 00000000..2ee4a74f --- /dev/null +++ b/kubernetes/common/alloy/configs/kubernetes/metrics/annotations-scrape.alloy @@ -0,0 +1,606 @@ +/* +Module Components: annotations_scrape +Description: Scrapes targets for metrics based on kubernetes annotations + +Note: Every argument except for "forward_to" is optional, and does have a defined default value. However, the values for these + arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... "). + This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will + does not override the value passed in, where coalesce() will return the first non-null value. + +Kubernetes Annotation Auto-Scraping +------------------------------------------------------------------------------------------------------------------------------------ +This module is meant to be used to automatically scrape targets based on a certain role and set of annotations. This module can be consumed +multiple times with different roles. The supported roles are: + + - pod + - service + - endpoints + +Typically, if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality you would use role="endpoints", if +mimicking the behavior of the PodMonitor functionality you would use role="pod". It is important to understand that with endpoints, +the target is typically going to be a pod, and whatever annotations that are set on the service will automatically be propagated to the +endpoints. This is why the role "endpoints" is used, because it will scrape the pod, but also consider the service annotations. Using +role="endpoints", which scrape each endpoint associated to the service. If role="service" is used, it will only scrape the service, only +hitting one of the endpoints associated to the service. + +This is where you must consider your scraping strategy, for example if you scrape a service like "kube-state-metrics" using +role="endpoints" you should only have a single replica of the kube-state-metrics pod, if you have multiple replicas, you should use +role="service" or a separate non-annotation job completely. Scraping a service instead of endpoints, is typically a rare use case, but +it is supported. + +There are other considerations for using annotation based scraping as well, which is metric relabeling rules that happen post scrape. If +you have a target that you want to apply a bunch of relabelings to or a very large metrics response payload, performance wise it will be +better to have a separate job for that target, rather than using use annotations. As every targert will go through the ssame relabeling. +Typical deployment strategies/options would be: + +Option #1 (recommended): + - Annotation Scraping for role="endpoints" + - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, etc.) or large metric payloads + - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, kube-scheduler, etc.) + +Option #2: + - Annotation Scraping for role="pod" + - Annotation Scraping for role="service" (i.e. kube-state-metrics, node-exporter, etc.) + - Separate Jobs for specific use cases or large metric payloads + - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, kube-scheduler, etc.) + +At no point should you use role="endpoints" and role="pod" together, as this will result in duplicate targets being scraped, thus +generating duplicate metrics. If you want to scrape both the pod and the service, use Option #2. + +Each port attached to an service/pod/endpoint is an eligible target, oftentimes it will have multiple ports. +There may be instances when you want to scrape all ports or some ports and not others. To support this +the following annotations are available: + + metrics.grafana.com/scrape: true + +the default scraping scheme is http, this can be specified as a single value which would override, the schema being used for all +ports attached to the target: + + metrics.grafana.com/scheme: https + +the default path to scrape is /metrics, this can be specified as a single value which would override, the scrape path being used +for all ports attached to the target: + + metrics.grafana.com/path: /metrics/some_path + +the default port to scrape is the target port, this can be specified as a single value which would override the scrape port being +used for all ports attached to the target, note that even if aan target had multiple targets, the relabel_config targets are +deduped before scraping: + + metrics.grafana.com/port: 8080 + +the default interval to scrape is 1m, this can be specified as a single value which would override, the scrape interval being used +for all ports attached to the target: + + metrics.grafana.com/interval: 5m + +the default timeout for scraping is 10s, this can be specified as a single value which would override, the scrape interval being +used for all ports attached to the target: + + metrics.grafana.com/timeout: 30s + +the default job is namespace/{{ service name }} or namespace/{{ controller_name }} depending on the role, there may be instances +in which a different job name is required because of a set of dashboards, rules, etc. to support this there is a job annotation +which will override the default value: + + metrics.grafana.com/job: integrations/kubernetes/kube-state-metrics +*/ + +declare "annotations_scrape" { + + /***************************************************************** + * ARGUMENTS + *****************************************************************/ + argument "forward_to" { + comment = "Must be a list(MetricssReceiver) where collected metrics should be forwarded to" + } + + argument "tenant" { + comment = "The tenant to filter metrics to. This does not have to be the tenantId, this is the value to look for in the metrics.agent.grafana.com/tenant annotation, and this can be a regex." + optional = true + } + + argument "cluster" { } + + argument "annotation_prefix" { + comment = "The annotation_prefix to use (default: metrics.grafana.com)" + default = "metrics.grafana.com" + optional = true + } + + argument "role" { + comment = "The role to use when looking for targets to scrape via annotations, can be: endpoints, service, pod (default: endpoints)" + optional = true + } + + argument "__sd_annotation" { + optional = true + comment = "The logic is used to transform the annotation argument into a valid label name by removing unsupported characters." + default = replace(replace(replace(coalesce(argument.annotation_prefix.value, "metrics.grafana.com"), ".", "_"), "/", "_"), "-", "_") + } + + argument "__pod_role" { + comment = "Most annotation targets service or pod that is all you want, however if the role is endpoints you want the pod" + optional = true + default = replace(coalesce(argument.role.value, "endpoints"), "endpoints", "pod") + } + + argument "__service_role" { + comment = "Most annotation targets service or pod that is all you want, however if the role is endpoints you we also want to consider service annotations" + optional = true + default = replace(coalesce(argument.role.value, "endpoints"), "endpoints", "service") + } + + argument "scrape_port_named_metrics" { + comment = "Whether or not to automatically scrape endpoints that have a port with 'metrics' in the name" + optional = true + default = false + } + + argument "keep_metrics" { + comment = "A regex of metrics to keep (default: (.+))" + optional = true + } + + argument "drop_metrics" { + comment = "A regex of metrics to drop (default: \"\")" + optional = true + } + + argument "scrape_interval" { + comment = "How often to scrape metrics from the targets (default: 60s)" + optional = true + } + + argument "scrape_timeout" { + comment = "How long before a scrape times out (default: 10s)" + optional = true + } + + /***************************************************************** + * Targets From Docker Discovery + *****************************************************************/ + discovery.kubernetes "annotation_metrics" { + role = coalesce(argument.role.value, "endpoints") + + selectors { + role = coalesce(argument.role.value, "endpoints") + } + } + + /***************************************************************** + * Discovery Relabelings (pre-scrape) + *****************************************************************/ + // filter metrics by kubernetes annotations + discovery.relabel "annotation_metrics_filter" { + targets = discovery.kubernetes.annotation_metrics.targets + + /**************************************************************************************************************** + * Handle Targets to Keep or Drop + ****************************************************************************************************************/ + // allow resources to declare their metrics scraped or not + // Example Annotation: + // metrics.grafana.com/scrape: false + // + // the label prometheus.io/service-monitor: "false" is a common label for headless services, when performing endpoint + // service discovery, if there is both a load-balanced service and headless service, this can result in duplicate + // scrapes if the name of the service is attached as a label. any targets with this label or annotation set should be dropped + rule { + action = "replace" + replacement = "false" + target_label = "__tmp_scrape" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_scrape", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_scrape", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_prometheus_io_service_monitor", + ] + separator = ";" + // only allow empty or true, otherwise defaults to false + regex = "^(?:;*)?(true)(;|true)*$" + replacement = "$1" + target_label = "__tmp_scrape" + } + + // add a __tmp_scrape_port_named_metrics from the argument.scrape_port_named_metrics + rule { + replacement = format("%t", argument.scrape_port_named_metrics.value) + target_label = "__tmp_scrape_port_named_metrics" + } + + // only keep targets that have scrape: true or "metrics" in the port name if the argument scrape_port_named_metrics + rule { + action = "keep" + source_labels = [ + "__tmp_scrape", + "__tmp_scrape_port_named_metrics", + // endpoints is the role and most meta labels started with "endpoints", however the port name is an exception and starts with "endpoint" + "__meta_kubernetes_" + replace(coalesce(argument.role.value, "endpoints"), "endpoints", "endpoint") + "_port_name", + ] + separator = ";" + regex = "^(true;.*|(|true);true;(.*metrics.*))$" + } + + // only keep targets where the pod is running or the pod_phase is empty and is not an init container. This will only exist for role="pod" or + // potentially role="endpoints", if it is a service the value is empty and thus allowed to pass, if it is an endpoint but not associated to a + // pod but rather a static IP or hostname, that could be outside of kubernetes allow endpoints to declare what tenant their metrics should be + // written to + rule { + action = "keep" + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "^(?i)(Running|)$" + } + + rule { + action = "keep" + source_labels = ["__meta_kubernetes_pod_ready"] + regex = "^(true|)$" + } + // if the container is an init container, drop it + rule { + action = "drop" + source_labels = ["__meta_kubernetes_pod_container_init"] + regex = "^(true)$" + } + + // allow resources to declare their metrics the tenant their metrics should be sent to, + // Example Annotation: + // metrics.grafana.com/tenant: primary + // + // Note: This does not necessarily have to be the actual tenantId, it can be a friendly name as well that is simply used + // to determine if the metrics should be gathered for the current tenant + rule { + action = "keep" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_tenant", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_tenant", + ] + regex = "^(" + coalesce(argument.tenant.value, ".*") + ")$" + } + + /**************************************************************************************************************** + * Handle Setting Scrape Metadata i.e. path, port, interval etc. + ****************************************************************************************************************/ + // allow resources to declare the protocol to use when collecting metrics, the default value is "http", + // Example Annotation: + // metrics.grafana.com/scheme: http + rule { + action = "replace" + replacement = "http" + target_label = "__scheme__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_scheme", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_scheme", + ] + separator = ";" + regex = "^(?:;*)?(https?).*$" + replacement = "$1" + target_label = "__scheme__" + } + + // allow resources to declare the port to use when collecting metrics, the default value is the discovered port from + // Example Annotation: + // metrics.grafana.com/port: 9090 + rule { + action = "replace" + source_labels = [ + "__address__", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_port", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_port", + ] + separator = ";" + regex = "^([^:]+)(?::\\d+)?;(\\d+)$" + replacement = "$1:$2" + target_label = "__address__" + } + + // allow resources to declare their the path to use when collecting their metrics, the default value is "/metrics", + // Example Annotation: + // metrics.grafana.com/path: /metrics/foo + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_path", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__metrics_path__" + } + + // allow resources to declare how often their metrics should be collected, the default value is 1m, + // the following duration formats are supported (s|m|ms|h|d): + // Example Annotation: + // metrics.grafana.com/interval: 5m + rule { + action = "replace" + replacement = coalesce(argument.scrape_interval.value, "60s") + target_label = "__scrape_interval__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_interval", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_interval", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_interval__" + } + + // allow resources to declare the timeout of the scrape request, the default value is 10s, + // the following duration formats are supported (s|m|ms|h|d): + // Example Annotation: + // metrics.grafana.com/timeout: 30s + rule { + action = "replace" + replacement = coalesce(argument.scrape_timeout.value, "10s") + target_label = "__scrape_timeout__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_timeout", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_timeout", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_timeout__" + } + + /**************************************************************************************************************** + * Handle Setting Common Labels + ****************************************************************************************************************/ + // set a source label + rule { + action = "replace" + replacement = "kubernetes" + target_label = "source" + } + + // set the cluster label + rule { + action = "replace" + replacement = argument.cluster.value + target_label = "cluster" + } + + // set the namespace label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + // set the target name label i.e. service name, pod name, etc. + // if the role is endpoints, the first valued field is used which would be __meta_kubernetes_pod_name, if the pod name is empty + // then the endpoint name would be used + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.__pod_role.value + "_name", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_name", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = argument.__pod_role.value + } + + // set a default job label to be the namespace/pod_controller_name or namespace/service_name + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_namespace", + "__meta_kubernetes_pod_controller_name", + argument.__pod_role.value, + ] + separator = ";" + regex = "^([^;]+)(?:;*)?([^;]+).*$" + replacement = "$1/$2" + target_label = "job" + } + + // if the controller is a ReplicaSet, drop the hash from the end of the ReplicaSet + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_pod_controller_type", + "__meta_kubernetes_namespace", + "__meta_kubernetes_pod_controller_name", + ] + separator = ";" + regex = "^(?:ReplicaSet);([^;]+);([^;]+)-.+$" + replacement = "$1/$2" + target_label = "job" + } + + // allow resources to declare their the job label value to use when collecting their metrics, the default value is "", + // Example Annotation: + // metrics.grafana.com/job: integrations/kubernetes/cadvisor + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_annotation_" + argument.__sd_annotation.value + "_job", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_job", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "job" + } + + // set the app name if specified as metadata labels "app:" or "app.kubernetes.io/name:" + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_app_kubernetes_io_name", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_k8s_app", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_app", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_app_kubernetes_io_name", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_k8s_app", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_app", + "__meta_kubernetes_" + argument.__service_role.value + "_label_app_kubernetes_io_name", + "__meta_kubernetes_" + argument.__service_role.value + "_label_k8s_app", + "__meta_kubernetes_" + argument.__service_role.value + "_label_app", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "app" + } + + // set the app component if specified as metadata labels "component:" or "app.kubernetes.io/component:" + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_app_kubernetes_io_component", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_component", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_app_kubernetes_io_component", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_component", + "__meta_kubernetes_" + argument.__service_role.value + "_label_app_kubernetes_io_component", + "__meta_kubernetes_" + argument.__service_role.value + "_label_component", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "component" + } + + // set the version if specified as metadata labels "version:" or "app.kubernetes.io/version:" or "app_version:" + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_app_kubernetes_io_version", + "__meta_kubernetes_" + coalesce(argument.role.value, "endpoints") + "_label_version", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_app_kubernetes_io_version", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_version", + "__meta_kubernetes_" + argument.__service_role.value + "_label_app_kubernetes_io_version", + "__meta_kubernetes_" + argument.__service_role.value + "_label_version", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "version" + } + + // set a workload label if the resource is a pod + // example: grafana-agent-68nv9 becomes DaemonSet/grafana-agent + rule { + source_labels = [ + "__meta_kubernetes_pod_controller_kind", + "__meta_kubernetes_pod_controller_name", + ] + separator = ";" + regex = "(.+);(.+)" + replacement = "$1/$2" + target_label = "workload" + } + // remove the hash from the ReplicaSet + rule { + source_labels = ["workload"] + regex = "(ReplicaSet/.+)-.+" + target_label = "workload" + } + } + + // only keep http targets + discovery.relabel "http_annotations" { + targets = discovery.relabel.annotation_metrics_filter.output + + rule { + action = "keep" + source_labels = ["__scheme__"] + regex = "http" + } + } + + // only keep https targets + discovery.relabel "https_annotations" { + targets = discovery.relabel.annotation_metrics_filter.output + + rule { + action = "keep" + source_labels = ["__scheme__"] + regex = "https" + } + } + + /***************************************************************** + * Prometheus Scrape Labels Targets + *****************************************************************/ + // scrape http only targtetsa + prometheus.scrape "http_annotations" { + targets = discovery.relabel.http_annotations.output + + job_name = "annotation-metrics-http" + scheme = "http" + scrape_interval = coalesce(argument.scrape_interval.value, "60s") + scrape_timeout = coalesce(argument.scrape_timeout.value, "10s") + + enable_protobuf_negotiation = true + scrape_classic_histograms = true + + clustering { + enabled = true + } + + forward_to = [prometheus.relabel.annotations.receiver] + } + + // scrape https only targtets + prometheus.scrape "https_annotations" { + targets = discovery.relabel.https_annotations.output + + job_name = "annotation-metrics-https" + scheme = "https" + scrape_interval = coalesce(argument.scrape_interval.value, "60s") + scrape_timeout = coalesce(argument.scrape_timeout.value, "10s") + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + tls_config { + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + insecure_skip_verify = false + server_name = "kubernetes" + } + + enable_protobuf_negotiation = true + scrape_classic_histograms = true + + clustering { + enabled = true + } + + forward_to = [prometheus.relabel.annotations.receiver] + } + + /***************************************************************** + * Prometheus Metric Relabelings (post-scrape) + *****************************************************************/ + // perform generic relabeling using keep_metrics and drop_metrics + prometheus.relabel "annotations" { + forward_to = argument.forward_to.value + // keep only metrics that match the keep_metrics regex + rule { + action = "keep" + source_labels = ["__name__"] + regex = coalesce(argument.keep_metrics.value, "(.+)") + } + + // drop metrics that match the drop_metrics regex + rule { + action = "drop" + source_labels = ["__name__"] + regex = coalesce(argument.drop_metrics.value, "") + } + } +} diff --git a/kubernetes/common/alloy/configs/kubernetes/metrics/podmonitors-scrape.alloy b/kubernetes/common/alloy/configs/kubernetes/metrics/podmonitors-scrape.alloy index 540e2ff0..100eff2a 100644 --- a/kubernetes/common/alloy/configs/kubernetes/metrics/podmonitors-scrape.alloy +++ b/kubernetes/common/alloy/configs/kubernetes/metrics/podmonitors-scrape.alloy @@ -13,6 +13,18 @@ declare "podmonitors_scrape" { optional = false } + argument "cluster" { } + + argument "keep_metrics" { + comment = "A regex of metrics to keep (default: (.+))" + optional = true + } + + argument "drop_metrics" { + comment = "A regex of metrics to drop (default: \"\")" + optional = true + } + argument "scrape_interval" { comment = "How often to scrape metrics from the targets (default: 60s)" optional = true @@ -27,7 +39,7 @@ declare "podmonitors_scrape" { * Kubernetes Auto Scrape PodMonitors *****************************************************************/ prometheus.operator.podmonitors "scrape" { - forward_to = argument.forward_to.value + forward_to = [prometheus.relabel.podmonitors.receiver] scrape { default_scrape_interval = coalesce(argument.scrape_interval.value, "60s") @@ -46,4 +58,33 @@ declare "podmonitors_scrape" { // } // } } + + /***************************************************************** + * Prometheus Metric Relabelings (post-scrape) + *****************************************************************/ + // perform generic relabeling using keep_metrics and drop_metrics + prometheus.relabel "podmonitors" { + forward_to = argument.forward_to.value + + // set the cluster label + rule { + action = "replace" + replacement = argument.cluster.value + target_label = "cluster" + } + + // keep only metrics that match the keep_metrics regex + rule { + action = "keep" + source_labels = ["__name__"] + regex = coalesce(argument.keep_metrics.value, "(.+)") + } + + // drop metrics that match the drop_metrics regex + rule { + action = "drop" + source_labels = ["__name__"] + regex = coalesce(argument.drop_metrics.value, "") + } + } } diff --git a/kubernetes/common/alloy/configs/kubernetes/metrics/rules-to-mimir.alloy b/kubernetes/common/alloy/configs/kubernetes/metrics/rules-to-mimir.alloy new file mode 100644 index 00000000..3dbd975a --- /dev/null +++ b/kubernetes/common/alloy/configs/kubernetes/metrics/rules-to-mimir.alloy @@ -0,0 +1,40 @@ +/* +Module Components: rules_to_mimir +Description: Auto discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance. +*/ + +declare "rules_to_mimir" { + + /***************************************************************** + * ARGUMENTS + *****************************************************************/ + argument "address" { + comment = "URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)" + optional = true + } + + argument "tenant" { + comment = "Mimir tenant ID. (default: anonymous)" + optional = true + } + + /******************************************** + * Kubernetes Prometheus Rules To Mimir + ********************************************/ + mimir.rules.kubernetes "rules_to_mimir" { + address = coalesce(argument.address.value, "http://nginx.gateway.svc:8080") + tenant_id = coalesce(argument.tenant.value, "anonymous") + + // rule_namespace_selector { + // match_labels = { + // auto_rules_to_mimir= "true", + // } + // } + + // rule_selector { + // match_labels = { + // auto_rules_to_mimir= "true", + // } + // } + } +} diff --git a/kubernetes/common/alloy/configs/kubernetes/metrics/servicemonitors-scrape.alloy b/kubernetes/common/alloy/configs/kubernetes/metrics/servicemonitors-scrape.alloy index d221e276..319e1423 100644 --- a/kubernetes/common/alloy/configs/kubernetes/metrics/servicemonitors-scrape.alloy +++ b/kubernetes/common/alloy/configs/kubernetes/metrics/servicemonitors-scrape.alloy @@ -13,6 +13,18 @@ declare "servicemonitors_scrape" { optional = false } + argument "cluster" { } + + argument "keep_metrics" { + comment = "A regex of metrics to keep (default: (.+))" + optional = true + } + + argument "drop_metrics" { + comment = "A regex of metrics to drop (default: \"\")" + optional = true + } + argument "scrape_interval" { comment = "How often to scrape metrics from the targets (default: 60s)" optional = true @@ -27,7 +39,7 @@ declare "servicemonitors_scrape" { * Kubernetes Auto Scrape ServiceMonitors *****************************************************************/ prometheus.operator.servicemonitors "scrape" { - forward_to = argument.forward_to.value + forward_to = [prometheus.relabel.servicemonitors.receiver] scrape { default_scrape_interval = coalesce(argument.scrape_interval.value, "60s") @@ -38,4 +50,33 @@ declare "servicemonitors_scrape" { enabled = true } } + + /***************************************************************** + * Prometheus Metric Relabelings (post-scrape) + *****************************************************************/ + // perform generic relabeling using keep_metrics and drop_metrics + prometheus.relabel "servicemonitors" { + forward_to = argument.forward_to.value + + // set the cluster label + rule { + action = "replace" + replacement = argument.cluster.value + target_label = "cluster" + } + + // keep only metrics that match the keep_metrics regex + rule { + action = "keep" + source_labels = ["__name__"] + regex = coalesce(argument.keep_metrics.value, "(.+)") + } + + // drop metrics that match the drop_metrics regex + rule { + action = "drop" + source_labels = ["__name__"] + regex = coalesce(argument.drop_metrics.value, "") + } + } } diff --git a/kubernetes/common/alloy/configs/kubernetes/profiles/annotations-scrape.alloy b/kubernetes/common/alloy/configs/kubernetes/profiles/annotations-scrape.alloy index 33d82ce5..05154c7d 100644 --- a/kubernetes/common/alloy/configs/kubernetes/profiles/annotations-scrape.alloy +++ b/kubernetes/common/alloy/configs/kubernetes/profiles/annotations-scrape.alloy @@ -13,6 +13,8 @@ declare "annotations_scrape" { comment = "Must be a list(ProfilessReceiver) where collected logs should be forwarded to" } + argument "cluster" { } + discovery.kubernetes "pyroscope_kubernetes" { role = "pod" } @@ -47,6 +49,13 @@ declare "annotations_scrape" { regex = "__meta_kubernetes_pod_label_(.+)" } + // set the cluster label + rule { + action = "replace" + replacement = argument.cluster.value + target_label = "cluster" + } + rule { action = "replace" source_labels = ["__meta_kubernetes_namespace"] diff --git a/kubernetes/common/alloy/configs/kubernetes/traces/process-and-transform.alloy b/kubernetes/common/alloy/configs/kubernetes/traces/process-and-transform.alloy index 7ab57c49..15d206f8 100644 --- a/kubernetes/common/alloy/configs/kubernetes/traces/process-and-transform.alloy +++ b/kubernetes/common/alloy/configs/kubernetes/traces/process-and-transform.alloy @@ -14,6 +14,8 @@ declare "process_and_transform" { comment = "Must be a list(TracesReceiver) where collected traces should be forwarded to" } + argument "cluster" { } + argument "logs_forward_to" { comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" } @@ -22,11 +24,6 @@ declare "process_and_transform" { comment = "Must be a list(MetricsReceiver) where collected metrics should be forwarded to" } - argument "cluster" { - optional = true - default = "k3d-k3s-codelab" - } - argument "otlp_http_endpoint" { optional = true default = "0.0.0.0:4318" @@ -60,9 +57,7 @@ declare "process_and_transform" { } output { - metrics = [otelcol.processor.batch.default.input] - logs = [otelcol.processor.resourcedetection.default.input] - traces = [otelcol.processor.resourcedetection.default.input] + traces = [otelcol.processor.k8sattributes.default.input] } } @@ -70,6 +65,10 @@ declare "process_and_transform" { * Otelcol for Metrics Logs Traces *****************************************************************/ otelcol.receiver.otlp "default" { + debug_metrics { + disable_high_cardinality_metrics = true + } + grpc { endpoint = argument.otlp_grpc_endpoint.value } @@ -79,7 +78,7 @@ declare "process_and_transform" { } output { - metrics = [otelcol.processor.batch.default.input] + metrics = [otelcol.processor.resourcedetection.default.input] logs = [otelcol.processor.resourcedetection.default.input] traces = [ otelcol.processor.resourcedetection.default.input, @@ -89,14 +88,36 @@ declare "process_and_transform" { } otelcol.processor.resourcedetection "default" { - detectors = ["env"] + detectors = ["env", "system"] + + system { + hostname_sources = ["os"] + } output { + metrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input] + logs = [otelcol.processor.k8sattributes.default.input] traces = [otelcol.processor.k8sattributes.default.input] } } + otelcol.processor.transform "add_metric_datapoint_attributes" { + error_mode = "ignore" + + metric_statements { + context = "datapoint" + statements = [ + "set(attributes[\"deployment.environment\"], resource.attributes[\"deployment.environment\"])", + "set(attributes[\"service.version\"], resource.attributes[\"service.version\"])", + ] + } + + output { + metrics = [otelcol.processor.k8sattributes.default.input] + } + } + otelcol.processor.k8sattributes "default" { extract { metadata = [ @@ -120,14 +141,44 @@ declare "process_and_transform" { } output { - logs = [otelcol.processor.transform.add_resource_attributes.input] - traces = [otelcol.processor.transform.add_resource_attributes.input] + metrics = [otelcol.processor.transform.default.input] + logs = [otelcol.processor.transform.default.input] + traces = [ + otelcol.processor.transform.default.input, + otelcol.connector.host_info.default.input, + ] } } - otelcol.processor.transform "add_resource_attributes" { + otelcol.connector.host_info "default" { + host_identifiers = ["k8s.node.name"] + + output { + metrics = [otelcol.processor.batch.host_info_batch.input] + } + } + + otelcol.processor.batch "host_info_batch" { + output { + metrics = [otelcol.exporter.prometheus.host_info_metrics.input] + } + } + + otelcol.exporter.prometheus "host_info_metrics" { + add_metric_suffixes = false + forward_to = argument.metrics_forward_to.value + } + + otelcol.processor.transform "default" { error_mode = "ignore" + metric_statements { + context = "resource" + statements = [ + `set(attributes["k8s.cluster.name"], "k3d-k3s-codelab") where attributes["k8s.cluster.name"] == nil`, + ] + } + log_statements { context = "resource" statements = [ @@ -141,22 +192,42 @@ declare "process_and_transform" { trace_statements { context = "resource" statements = [ + `limit(attributes, 100, [])`, + `truncate_all(attributes, 4096)`, `set(attributes["k8s.cluster.name"], "k3d-k3s-codelab") where attributes["k8s.cluster.name"] == nil`, ] } + trace_statements { + context = "span" + statements = [ + `limit(attributes, 100, [])`, + `truncate_all(attributes, 4096)`, + ] + } + output { - logs = [otelcol.processor.filter.default.input] - traces = [otelcol.processor.filter.default.input] + metrics = [otelcol.processor.filter.default.input] + logs = [otelcol.processor.filter.default.input] + traces = [otelcol.processor.filter.default.input] } } otelcol.processor.filter "default" { error_mode = "ignore" + traces { + span = [ + "attributes[\"http.route\"] == \"/live\"", + "attributes[\"http.route\"] == \"/healthy\"", + "attributes[\"http.route\"] == \"/ready\"", + ] + } + output { - logs = [otelcol.processor.batch.default.input] - traces = [otelcol.processor.batch.default.input] + metrics = [otelcol.processor.batch.default.input] + logs = [otelcol.processor.batch.default.input] + traces = [otelcol.processor.batch.default.input] } } diff --git a/kubernetes/common/alloy/kustomization.yaml b/kubernetes/common/alloy/kustomization.yaml index 85bd3691..779f3945 100644 --- a/kubernetes/common/alloy/kustomization.yaml +++ b/kubernetes/common/alloy/kustomization.yaml @@ -22,7 +22,9 @@ configMapGenerator: - name: alloy-modules-kubernetes-metrics namespace: monitoring-system files: + - configs/kubernetes/metrics/annotations-scrape.alloy - configs/kubernetes/metrics/podmonitors-scrape.alloy + - configs/kubernetes/metrics/rules-to-mimir.alloy - configs/kubernetes/metrics/servicemonitors-scrape.alloy - name: alloy-modules-kubernetes-logs namespace: monitoring-system @@ -30,6 +32,7 @@ configMapGenerator: - configs/kubernetes/logs/annotations-scrape.alloy - configs/kubernetes/logs/k8s-events.alloy - configs/kubernetes/logs/keep-labels.alloy + - configs/kubernetes/logs/rules-to-loki.alloy - name: alloy-modules-kubernetes-traces namespace: monitoring-system files: diff --git a/kubernetes/common/alloy/manifests/k8s-all-in-one.yaml b/kubernetes/common/alloy/manifests/k8s-all-in-one.yaml index df229540..e2b617ef 100644 --- a/kubernetes/common/alloy/manifests/k8s-all-in-one.yaml +++ b/kubernetes/common/alloy/manifests/k8s-all-in-one.yaml @@ -144,28 +144,25 @@ apiVersion: v1 data: annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: Scrapes targets for logs based on kubernetes Pod annotations\n\n Annotations:\n - \ logs.grafana.com/ingest: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare + \ logs.grafana.com/scrape: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this - can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\t// arguments - for kubernetes discovery\n\targument \"namespaces\" {\n\t\tcomment = \"The namespaces - to look for targets in (default: [\\\"kube-system\\\"] is all namespaces)\"\n\t\toptional - = true\n\t}\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\targument + \"cluster\" { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix to use (default: logs.grafana.com)\"\n\t\tdefault = \"logs.grafana.com\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional = true\n\t\tcomment \ = \"The logic is used to transform the annotation argument into a valid label name by removing unsupported characters.\"\n\t\tdefault = replace(replace(replace(coalesce(argument.annotation_prefix.value, \"logs.grafana.com\"), \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\t// - find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\n\t\tnamespaces - {\n\t\t\tnames = coalesce(argument.namespaces.value, [])\n\t\t}\n\t}\n\n\t// filter - logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" + find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\t}\n\n\t// + filter logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" {\n\t\ttargets = discovery.kubernetes.annotation_logs.targets\n\n\t\t// allow pods to declare their logs to be ingested or not, the default is true\n\t\t// - \ i.e. logs.grafana.com/ingest: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + \ i.e. logs.grafana.com/scrape: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\n\t\t// allow pods to declare what tenant their logs should be written to, the following annotation @@ -175,6 +172,8 @@ data: + argument.tenant.value + \")$\"\n\t\t}\n\n\t\t// set the instance label as the name of the worker node the pod is on\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_node_name\"]\n\t\t\ttarget_label = \"instance\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\t// set the pod label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\t// @@ -239,12 +238,11 @@ data: ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\toptional - = true\n\t}\n\n\targument \"cluster\" {\n\t\toptional = true\n\t}\n\n\tloki.source.kubernetes_events - \"cluster_events\" {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format + = true\n\t}\n\n\targument \"cluster\" { }\n\n\tloki.source.kubernetes_events \"cluster_events\" + {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format = \"logfmt\"\n\t\tforward_to = [loki.process.logs_service.receiver]\n\t}\n\n\tloki.process \"logs_service\" {\n\t\tstage.static_labels {\n\t\t\tvalues = {\n\t\t\t\tcluster - = coalesce(argument.cluster.value, \"k3d\"),\n\t\t\t}\n\t\t}\n\t\tforward_to = - argument.forward_to.value\n\t}\n}\n" + = argument.cluster.value,\n\t\t\t}\n\t\t}\n\t\tforward_to = argument.forward_to.value\n\t}\n}\n" keep-labels.alloy: "/*\nModule Components: keep_labels\nDescription: Pre-defined set of labels to keep, this stage should always be in-place as the previous relabeing\n \ stages make every pod label and annotation a label in the pipeline, @@ -266,46 +264,392 @@ data: = argument.keep_labels.value\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* EXPORTS\n\t*****************************************************************/\n\texport \"receiver\" {\n\t\tvalue = loki.process.keep_labels.receiver\n\t}\n}\n" + rules-to-loki.alloy: "/*\nModule Components: rules_to_loki\nDescription: Auto discovers + PrometheusRule Kubernetes resources and loads them into a Loki instance.\n*/\n\ndeclare + \"rules_to_loki\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + fake)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Loki\n\t********************************************/\n\tloki.rules.kubernetes + \"rules_to_loki\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:3100\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_loki= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\trule_selector {\n\t\t\tmatch_labels = {\n\t\t\t\tauto_rules_to_loki + = \"true\",\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k namespace: monitoring-system --- apiVersion: v1 data: + annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: + Scrapes targets for metrics based on kubernetes annotations\n\nNote: Every argument + except for \"forward_to\" is optional, and does have a defined default value. + \ However, the values for these\n arguments are not defined using the default + = \" ... \" argument syntax, but rather using the coalesce(argument.value, \" + ... \").\n This is because if the argument passed in from another consuming + module is set to null, the default = \" ... \" syntax will\n does not override + the value passed in, where coalesce() will return the first non-null value.\n\nKubernetes + Annotation Auto-Scraping\n------------------------------------------------------------------------------------------------------------------------------------\nThis + module is meant to be used to automatically scrape targets based on a certain + role and set of annotations. This module can be consumed\nmultiple times with + different roles. The supported roles are:\n\n - pod\n - service\n - endpoints\n\nTypically, + if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality + you would use role=\"endpoints\", if\nmimicking the behavior of the PodMonitor + functionality you would use role=\"pod\". It is important to understand that + with endpoints,\nthe target is typically going to be a pod, and whatever annotations + that are set on the service will automatically be propagated to the\nendpoints. + \ This is why the role \"endpoints\" is used, because it will scrape the pod, + but also consider the service annotations. Using\nrole=\"endpoints\", which scrape + each endpoint associated to the service. If role=\"service\" is used, it will + only scrape the service, only\nhitting one of the endpoints associated to the + service.\n\nThis is where you must consider your scraping strategy, for example + if you scrape a service like \"kube-state-metrics\" using\nrole=\"endpoints\" + you should only have a single replica of the kube-state-metrics pod, if you have + multiple replicas, you should use\nrole=\"service\" or a separate non-annotation + job completely. Scraping a service instead of endpoints, is typically a rare + use case, but\nit is supported.\n\nThere are other considerations for using annotation + based scraping as well, which is metric relabeling rules that happen post scrape. + \ If\nyou have a target that you want to apply a bunch of relabelings to or a + very large metrics response payload, performance wise it will be\nbetter to have + a separate job for that target, rather than using use annotations. As every targert + will go through the ssame relabeling.\nTypical deployment strategies/options would + be:\n\nOption #1 (recommended):\n - Annotation Scraping for role=\"endpoints\"\n + \ - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, + etc.) or large metric payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, + kube-apiserver, kube-scheduler, etc.)\n\nOption #2:\n - Annotation Scraping for + role=\"pod\"\n - Annotation Scraping for role=\"service\" (i.e. kube-state-metrics, + node-exporter, etc.)\n - Separate Jobs for specific use cases or large metric + payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, + kube-scheduler, etc.)\n\nAt no point should you use role=\"endpoints\" and role=\"pod\" + together, as this will result in duplicate targets being scraped, thus\ngenerating + duplicate metrics. If you want to scrape both the pod and the service, use Option + #2.\n\nEach port attached to an service/pod/endpoint is an eligible target, oftentimes + it will have multiple ports.\nThere may be instances when you want to scrape all + ports or some ports and not others. To support this\nthe following annotations + are available:\n\n metrics.grafana.com/scrape: true\n\nthe default scraping scheme + is http, this can be specified as a single value which would override, the schema + being used for all\nports attached to the target:\n\n metrics.grafana.com/scheme: + https\n\nthe default path to scrape is /metrics, this can be specified as a single + value which would override, the scrape path being used\nfor all ports attached + to the target:\n\n metrics.grafana.com/path: /metrics/some_path\n\nthe default + port to scrape is the target port, this can be specified as a single value which + would override the scrape port being\nused for all ports attached to the target, + note that even if aan target had multiple targets, the relabel_config targets + are\ndeduped before scraping:\n\n metrics.grafana.com/port: 8080\n\nthe default + interval to scrape is 1m, this can be specified as a single value which would + override, the scrape interval being used\nfor all ports attached to the target:\n\n + \ metrics.grafana.com/interval: 5m\n\nthe default timeout for scraping is 10s, + this can be specified as a single value which would override, the scrape interval + being\nused for all ports attached to the target:\n\n metrics.grafana.com/timeout: + 30s\n\nthe default job is namespace/{{ service name }} or namespace/{{ controller_name + }} depending on the role, there may be instances\nin which a different job name + is required because of a set of dashboards, rules, etc. to support this there + is a job annotation\nwhich will override the default value:\n\n metrics.grafana.com/job: + integrations/kubernetes/kube-state-metrics\n*/\n\ndeclare \"annotations_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment + \ = \"The tenant to filter metrics to. This does not have to be the tenantId, + this is the value to look for in the metrics.agent.grafana.com/tenant annotation, + and this can be a regex.\"\n\t\toptional = true\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + to use (default: metrics.grafana.com)\"\n\t\tdefault = \"metrics.grafana.com\"\n\t\toptional + = true\n\t}\n\n\targument \"role\" {\n\t\tcomment = \"The role to use when looking + for targets to scrape via annotations, can be: endpoints, service, pod (default: + endpoints)\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional + = true\n\t\tcomment = \"The logic is used to transform the annotation argument + into a valid label name by removing unsupported characters.\"\n\t\tdefault = + replace(replace(replace(coalesce(argument.annotation_prefix.value, \"metrics.grafana.com\"), + \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\targument \"__pod_role\" + {\n\t\tcomment = \"Most annotation targets service or pod that is all you want, + however if the role is endpoints you want the pod\"\n\t\toptional = true\n\t\tdefault + \ = replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"pod\")\n\t}\n\n\targument + \"__service_role\" {\n\t\tcomment = \"Most annotation targets service or pod + that is all you want, however if the role is endpoints you we also want to consider + service annotations\"\n\t\toptional = true\n\t\tdefault = replace(coalesce(argument.role.value, + \"endpoints\"), \"endpoints\", \"service\")\n\t}\n\n\targument \"scrape_port_named_metrics\" + {\n\t\tcomment = \"Whether or not to automatically scrape endpoints that have + a port with 'metrics' in the name\"\n\t\toptional = true\n\t\tdefault = false\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: (.+))\"\n\t\toptional + = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment = \"A regex of metrics + to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" + {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + Targets From Docker Discovery\n\t*****************************************************************/\n\tdiscovery.kubernetes + \"annotation_metrics\" {\n\t\trole = coalesce(argument.role.value, \"endpoints\")\n\n\t\tselectors + {\n\t\t\trole = coalesce(argument.role.value, \"endpoints\")\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t*****************************************************************/\n\t// + filter metrics by kubernetes annotations\n\tdiscovery.relabel \"annotation_metrics_filter\" + {\n\t\ttargets = discovery.kubernetes.annotation_metrics.targets\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Targets to Keep or Drop\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare their metrics scraped or not\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/scrape: false\n\t\t//\n\t\t// the label prometheus.io/service-monitor: + \"false\" is a common label for headless services, when performing endpoint\n\t\t// + service discovery, if there is both a load-balanced service and headless service, + this can result in duplicate\n\t\t// scrapes if the name of the service is attached + as a label. any targets with this label or annotation set should be dropped\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"false\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, + \"endpoints\") + \"_label_prometheus_io_service_monitor\",\n\t\t\t]\n\t\t\tseparator + = \";\"\n\t\t\t// only allow empty or true, otherwise defaults to false\n\t\t\tregex + \ = \"^(?:;*)?(true)(;|true)*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\t// add a __tmp_scrape_port_named_metrics from + the argument.scrape_port_named_metrics\n\t\trule {\n\t\t\treplacement = format(\"%t\", + argument.scrape_port_named_metrics.value)\n\t\t\ttarget_label = \"__tmp_scrape_port_named_metrics\"\n\t\t}\n\n\t\t// + only keep targets that have scrape: true or \"metrics\" in the port name if the + argument scrape_port_named_metrics\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__tmp_scrape\",\n\t\t\t\t\"__tmp_scrape_port_named_metrics\",\n\t\t\t\t// + endpoints is the role and most meta labels started with \"endpoints\", however + the port name is an exception and starts with \"endpoint\"\n\t\t\t\t\"__meta_kubernetes_\" + + replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"endpoint\") + + \"_port_name\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(true;.*|(|true);true;(.*metrics.*))$\"\n\t\t}\n\n\t\t// + only keep targets where the pod is running or the pod_phase is empty and is not + an init container. This will only exist for role=\"pod\" or\n\t\t// potentially + role=\"endpoints\", if it is a service the value is empty and thus allowed to + pass, if it is an endpoint but not associated to a\n\t\t// pod but rather a static + IP or hostname, that could be outside of kubernetes allow endpoints to declare + what tenant their metrics should be\n\t\t// written to\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex + \ = \"^(?i)(Running|)$\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__meta_kubernetes_pod_ready\"]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\t\t// + if the container is an init container, drop it\n\t\trule {\n\t\t\taction = + \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_container_init\"]\n\t\t\tregex + \ = \"^(true)$\"\n\t\t}\n\n\t\t// allow resources to declare their metrics + the tenant their metrics should be sent to,\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/tenant: primary\n\t\t//\n\t\t// Note: This does not necessarily + have to be the actual tenantId, it can be a friendly name as well that is simply + used\n\t\t// to determine if the metrics should be gathered for the current + tenant\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_tenant\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_tenant\",\n\t\t\t]\n\t\t\tregex + = \"^(\" + coalesce(argument.tenant.value, \".*\") + \")$\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Scrape Metadata i.e. path, port, interval etc.\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare the protocol to use when collecting metrics, the default + value is \"http\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/scheme: + http\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"http\"\n\t\t\ttarget_label + = \"__scheme__\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scheme\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scheme\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?(https?).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scheme__\"\n\t\t}\n\n\t\t// allow resources + to declare the port to use when collecting metrics, the default value is the discovered + port from\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/port: 9090\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__address__\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_port\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_port\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^:]+)(?::\\\\d+)?;(\\\\d+)$\"\n\t\t\treplacement + \ = \"$1:$2\"\n\t\t\ttarget_label = \"__address__\"\n\t\t}\n\n\t\t// allow resources + to declare their the path to use when collecting their metrics, the default value + is \"/metrics\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/path: + /metrics/foo\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_path\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_path\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__metrics_path__\"\n\t\t}\n\n\t\t// allow resources + to declare how often their metrics should be collected, the default value is 1m,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/interval: 5m\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_interval\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_interval\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\t// allow + resources to declare the timeout of the scrape request, the default value is 10s,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/timeout: 30s\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_timeout\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_timeout\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Common Labels\n\t\t****************************************************************************************************************/\n\t\t// + set a source label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = \"kubernetes\"\n\t\t\ttarget_label = \"source\"\n\t\t}\n\n\t\t// set the cluster + label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = argument.cluster.value\n\t\t\ttarget_label + = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label + \ = \"namespace\"\n\t\t}\n\n\t\t// set the target name label i.e. service name, + pod name, etc.\n\t\t// if the role is endpoints, the first valued field is used + which would be __meta_kubernetes_pod_name, if the pod name is empty\n\t\t// then + the endpoint name would be used\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + argument.__pod_role.value + \"_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = + \"$1\"\n\t\t\ttarget_label = argument.__pod_role.value\n\t\t}\n\n\t\t// set a + default job label to be the namespace/pod_controller_name or namespace/service_name\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t\targument.__pod_role.value,\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^;]+)(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// if the controller + is a ReplicaSet, drop the hash from the end of the ReplicaSet\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_type\",\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:ReplicaSet);([^;]+);([^;]+)-.+$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// allow resources to + declare their the job label value to use when collecting their metrics, the default + value is \"\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/job: + integrations/kubernetes/cadvisor\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_job\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_job\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// set the app name if + specified as metadata labels \"app:\" or \"app.kubernetes.io/name:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"app\"\n\t\t}\n\n\t\t// + set the app component if specified as metadata labels \"component:\" or \"app.kubernetes.io/component:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_component\",\n\t\t\t]\n\t\t\tregex + \ = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"component\"\n\t\t}\n\n\t\t// set the version if specified as metadata labels + \"version:\" or \"app.kubernetes.io/version:\" or \"app_version:\"\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_version\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"version\"\n\t\t}\n\n\t\t// + set a workload label if the resource is a pod\n\t\t// example: grafana-agent-68nv9 + becomes DaemonSet/grafana-agent\n\t\trule {\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_kind\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"(.+);(.+)\"\n\t\t\treplacement = \"$1/$2\"\n\t\t\ttarget_label + = \"workload\"\n\t\t}\n\t\t// remove the hash from the ReplicaSet\n\t\trule {\n\t\t\tsource_labels + = [\"workload\"]\n\t\t\tregex = \"(ReplicaSet/.+)-.+\"\n\t\t\ttarget_label + \ = \"workload\"\n\t\t}\n\t}\n\n\t// only keep http targets\n\tdiscovery.relabel + \"http_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"http\"\n\t\t}\n\t}\n\n\t// only keep https targets\n\tdiscovery.relabel + \"https_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"https\"\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Scrape Labels Targets\n\t*****************************************************************/\n\t// + scrape http only targtetsa\n\tprometheus.scrape \"http_annotations\" {\n\t\ttargets + = discovery.relabel.http_annotations.output\n\n\t\tjob_name = \"annotation-metrics-http\"\n\t\tscheme + \ = \"http\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t// + scrape https only targtets\n\tprometheus.scrape \"https_annotations\" {\n\t\ttargets + = discovery.relabel.https_annotations.output\n\n\t\tjob_name = \"annotation-metrics-https\"\n\t\tscheme + \ = \"https\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\tbearer_token_file + = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n\n\t\ttls_config {\n\t\t\tca_file + \ = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n\t\t\tinsecure_skip_verify + = false\n\t\t\tserver_name = \"kubernetes\"\n\t\t}\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"annotations\" {\n\t\tforward_to = argument.forward_to.value\n\t\t// keep only + metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(.+)\")\n\t\t}\n\n\t\t// drop metrics that match the drop_metrics regex\n\t\trule + {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex + \ = coalesce(argument.drop_metrics.value, \"\")\n\t\t}\n\t}\n}\n" podmonitors-scrape.alloy: "/*\nModule Components: podmonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.podmonitors\n*/\n\ndeclare \"podmonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape PodMonitors\n\t*****************************************************************/\n\tprometheus.operator.podmonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.podmonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\t// selector {\n\t\t// \tmatch_expression {\n\t\t// \t\tkey = \"team\"\n\t\t// \t\toperator = \"In\"\n\t\t// \t\tvalues - \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n}\n" + \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"podmonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set the + cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" + rules-to-mimir.alloy: "/*\nModule Components: rules_to_mimir\nDescription: Auto + discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance.\n*/\n\ndeclare + \"rules_to_mimir\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + anonymous)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Mimir\n\t********************************************/\n\tmimir.rules.kubernetes + \"rules_to_mimir\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:8080\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\t// rule_selector {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= + \"true\",\n\t\t// \t}\n\t\t// }\n\t}\n}\n" servicemonitors-scrape.alloy: "/*\nModule Components: servicemonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.servicemonitors\n*/\n\ndeclare \"servicemonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape ServiceMonitors\n\t*****************************************************************/\n\tprometheus.operator.servicemonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.servicemonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering - {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n" + {\n\t\t\tenabled = true\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"servicemonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set + the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f namespace: monitoring-system --- apiVersion: v1 @@ -315,19 +659,21 @@ data: \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(ProfilessReceiver) where collected - logs should be forwarded to\"\n\t}\n\n\tdiscovery.kubernetes \"pyroscope_kubernetes\" - {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape config allows to define - annotations based scraping.\n\t//\n\t// For example the following annotations:\n\t//\n\t// - ```\n\t// profiles.grafana.com/memory.scrape: \"true\"\n\t// profiles.grafana.com/memory.port: - \"8080\"\n\t// profiles.grafana.com/cpu.scrape: \"true\"\n\t// profiles.grafana.com/cpu.port: - \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: \"true\"\n\t// profiles.grafana.com/goroutine.port: - \"8080\"\n\t// ```\n\t//\n\t// will scrape the `memory`, `cpu` and `goroutine` - profiles from the `8080` port of the pod.\n\t//\n\t// For more information see - https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel + logs should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\tdiscovery.kubernetes + \"pyroscope_kubernetes\" {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape + config allows to define annotations based scraping.\n\t//\n\t// For example the + following annotations:\n\t//\n\t// ```\n\t// profiles.grafana.com/memory.scrape: + \"true\"\n\t// profiles.grafana.com/memory.port: \"8080\"\n\t// profiles.grafana.com/cpu.scrape: + \"true\"\n\t// profiles.grafana.com/cpu.port: \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: + \"true\"\n\t// profiles.grafana.com/goroutine.port: \"8080\"\n\t// ```\n\t//\n\t// + will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port + of the pod.\n\t//\n\t// For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel \"kubernetes_pods\" {\n\t\ttargets = concat(discovery.kubernetes.pyroscope_kubernetes.targets)\n\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex \ = \"Pending|Succeeded|Failed|Completed\"\n\t\t}\n\n\t\trule {\n\t\t\taction - = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\trule + = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\trule @@ -546,7 +892,7 @@ data: {\n\t\t\t\tenabled = true\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 namespace: monitoring-system --- apiVersion: v1 @@ -556,40 +902,58 @@ data: Transformation\ndeclare \"process_and_transform\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"traces_forward_to\" {\n\t\tcomment = \"Must be a list(TracesReceiver) where - collected traces should be forwarded to\"\n\t}\n\n\targument \"logs_forward_to\" - {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be - forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment = \"Must - be a list(MetricsReceiver) where collected metrics should be forwarded to\"\n\t}\n\n\targument - \"cluster\" {\n\t\toptional = true\n\t\tdefault = \"k3d-k3s-codelab\"\n\t}\n\n\targument - \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4318\"\n\t}\n\n\targument - \"otlp_grpc_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* + collected traces should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\targument + \"logs_forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected + logs should be forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment + = \"Must be a list(MetricsReceiver) where collected metrics should be forwarded + to\"\n\t}\n\n\targument \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault + \ = \"0.0.0.0:4318\"\n\t}\n\n\targument \"otlp_grpc_endpoint\" {\n\t\toptional + = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* Jaeger for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.jaeger \"default\" {\n\t\tprotocols {\n\t\t\tgrpc {\n\t\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t\t}\n\n\t\t\tthrift_http {\n\t\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t\t}\n\n\t\t\tthrift_binary {\n\t\t\t\tendpoint = \"0.0.0.0:6832\"\n\t\t\t}\n\n\t\t\tthrift_compact {\n\t\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [otelcol.processor.resourcedetection.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + {\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* Otelcol for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.otlp - \"default\" {\n\t\tgrpc {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp - {\n\t\t\tendpoint = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics - = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection - \"default\" {\n\t\tdetectors = [\"env\"]\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces - = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes + \"default\" {\n\t\tdebug_metrics {\n\t\t\tdisable_high_cardinality_metrics = true\n\t\t}\n\n\t\tgrpc + {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp {\n\t\t\tendpoint + = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.resourcedetection.default.input]\n\t\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection + \"default\" {\n\t\tdetectors = [\"env\", \"system\"]\n\n\t\tsystem {\n\t\t\thostname_sources + = [\"os\"]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\n\t\t\tlogs + \ = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform + \"add_metric_datapoint_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements + {\n\t\t\tcontext = \"datapoint\"\n\t\t\tstatements = [\n\t\t\t\t\"set(attributes[\\\"deployment.environment\\\"], + resource.attributes[\\\"deployment.environment\\\"])\",\n\t\t\t\t\"set(attributes[\\\"service.version\\\"], + resource.attributes[\\\"service.version\\\"])\",\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes \"default\" {\n\t\textract {\n\t\t\tmetadata = [\n\t\t\t\t\"k8s.namespace.name\",\n\t\t\t\t\"k8s.pod.name\",\n\t\t\t\t\"k8s.deployment.name\",\n\t\t\t\t\"k8s.statefulset.name\",\n\t\t\t\t\"k8s.daemonset.name\",\n\t\t\t\t\"k8s.cronjob.name\",\n\t\t\t\t\"k8s.job.name\",\n\t\t\t\t\"k8s.node.name\",\n\t\t\t\t\"k8s.pod.uid\",\n\t\t\t\t\"k8s.pod.start_time\",\n\t\t\t]\n\t\t}\n\n\t\tpod_association {\n\t\t\tsource {\n\t\t\t\tfrom = \"connection\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t\ttraces - = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform - \"add_resource_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tlog_statements + {\n\t\t\tmetrics = [otelcol.processor.transform.default.input]\n\t\t\tlogs = + [otelcol.processor.transform.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.transform.default.input,\n\t\t\t\totelcol.connector.host_info.default.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.connector.host_info + \"default\" {\n\t\thost_identifiers = [\"k8s.node.name\"]\n\n\t\toutput {\n\t\t\tmetrics + = [otelcol.processor.batch.host_info_batch.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + \"host_info_batch\" {\n\t\toutput {\n\t\t\tmetrics = [otelcol.exporter.prometheus.host_info_metrics.input]\n\t\t}\n\t}\n\n\totelcol.exporter.prometheus + \"host_info_metrics\" {\n\t\tadd_metric_suffixes = false\n\t\tforward_to = + argument.metrics_forward_to.value\n\t}\n\n\totelcol.processor.transform \"default\" + {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements {\n\t\t\tcontext = + \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\tlog_statements {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])`,\n\t\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t\t`set(attributes[\"loki.resource.labels\"], \"pod, namespace, cluster, job\")`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements - {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], - \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter - \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.batch.default.input]\n\t\t\ttraces - = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements + {\n\t\t\tcontext = \"span\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces + \ = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter + \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\ttraces {\n\t\t\tspan = [\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/live\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] == \\\"/healthy\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/ready\\\"\",\n\t\t\t]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs + \ = [otelcol.processor.batch.default.input]\n\t\t\ttraces = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch \"default\" {\n\t\tsend_batch_size = 16384\n\t\tsend_batch_max_size = 0\n\t\ttimeout \ = \"5s\"\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\t\tlogs \ = [otelcol.processor.memory_limiter.default.input]\n\t\t\ttraces = [otelcol.processor.memory_limiter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.memory_limiter @@ -625,7 +989,7 @@ data: \"alloy_traces_input\" {\n\t\tvalue = otelcol.processor.batch.default.input\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 namespace: monitoring-system --- apiVersion: v1 @@ -821,16 +1185,16 @@ spec: name: alloy-config-ftt29f8k85 name: config - configMap: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f name: modules-kubernetes-metrics - configMap: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k name: modules-kubernetes-logs - configMap: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 name: modules-kubernetes-traces - configMap: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 name: modules-kubernetes-profiles --- apiVersion: monitoring.coreos.com/v1 diff --git a/kubernetes/common/grafana/kustomization.yaml b/kubernetes/common/grafana/kustomization.yaml index 233e6e14..1f499643 100644 --- a/kubernetes/common/grafana/kustomization.yaml +++ b/kubernetes/common/grafana/kustomization.yaml @@ -40,7 +40,7 @@ secretGenerator: - name: grafana-secret literals: - admin-user=admin - - admin-password=admin_password + - admin-password=admin - name: grafana-env options: disableNameSuffixHash: true diff --git a/kubernetes/common/grafana/manifests/k8s-all-in-one.yaml b/kubernetes/common/grafana/manifests/k8s-all-in-one.yaml index a6d6b35f..f276537a 100644 --- a/kubernetes/common/grafana/manifests/k8s-all-in-one.yaml +++ b/kubernetes/common/grafana/manifests/k8s-all-in-one.yaml @@ -3522,11 +3522,11 @@ type: Opaque --- apiVersion: v1 data: - admin-password: YWRtaW5fcGFzc3dvcmQ= + admin-password: YWRtaW4= admin-user: YWRtaW4= kind: Secret metadata: - name: grafana-secret-55dh9ff969 + name: grafana-secret-hd8f4c4fbb namespace: monitoring-system type: Opaque --- @@ -3605,12 +3605,12 @@ spec: valueFrom: secretKeyRef: key: admin-user - name: grafana-secret-55dh9ff969 + name: grafana-secret-hd8f4c4fbb - name: REQ_PASSWORD valueFrom: secretKeyRef: key: admin-password - name: grafana-secret-55dh9ff969 + name: grafana-secret-hd8f4c4fbb - name: REQ_URL value: http://localhost:3000/api/admin/provisioning/dashboards/reload - name: REQ_METHOD @@ -3645,12 +3645,12 @@ spec: valueFrom: secretKeyRef: key: admin-user - name: grafana-secret-55dh9ff969 + name: grafana-secret-hd8f4c4fbb - name: REQ_PASSWORD valueFrom: secretKeyRef: key: admin-password - name: grafana-secret-55dh9ff969 + name: grafana-secret-hd8f4c4fbb - name: REQ_URL value: http://localhost:3000/api/admin/provisioning/datasources/reload - name: REQ_METHOD @@ -3677,12 +3677,12 @@ spec: valueFrom: secretKeyRef: key: admin-user - name: grafana-secret-55dh9ff969 + name: grafana-secret-hd8f4c4fbb - name: GF_SECURITY_ADMIN_PASSWORD valueFrom: secretKeyRef: key: admin-password - name: grafana-secret-55dh9ff969 + name: grafana-secret-hd8f4c4fbb - name: GF_PATHS_DATA value: /var/lib/grafana/ - name: GF_PATHS_LOGS diff --git a/kubernetes/microservices-mode/logs/k8s-all-in-one.yaml b/kubernetes/microservices-mode/logs/k8s-all-in-one.yaml index d577cac0..fa46c633 100644 --- a/kubernetes/microservices-mode/logs/k8s-all-in-one.yaml +++ b/kubernetes/microservices-mode/logs/k8s-all-in-one.yaml @@ -340,31 +340,27 @@ data: = coalesce(env(\"SELF_HOSTED_METRICS_ENDPOINT_URL\"), \"http://nginx.gateway.svc:8080/api/v1/push\")\n\tlogs_endpoint_url \ = coalesce(env(\"SELF_HOSTED_LOGS_ENDPOINT_URL\"), \"http://nginx.gateway.svc:3100/loki/api/v1/push\")\n}\n\n/********************************************\n * Logs\n ********************************************/\nimport.file \"logs\" {\n\tfilename - = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/logs\"\n}\n\nlogs.annotations_scrape - \"kubernetes\" {\n\tannotation_prefix = \"logs.grafana.com\"\n\tforward_to = - [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.keep_labels \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.logs_receiver]\n}\n\n/********************************************\n + = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/logs\"\n}\n\nlogs.kubernetes_cluster_events + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\n\tforward_to + = [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.annotations_scrape \"kubernetes\" + {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\n\tforward_to + = [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.keep_labels \"kubernetes\" + {\n\tforward_to = [provider.self_hosted_stack.kubernetes.logs_receiver]\n}\n\n/********************************************\n * Metrics\n ********************************************/\nimport.file \"metrics\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/metrics\"\n}\n\nmetrics.podmonitors_scrape \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape - \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\n/*****************************************************************\n* + + \"/kubernetes/metrics\"\n}\n\nmetrics.annotations_scrape \"kubernetes\" {\n\tcluster + \ = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\n/*****************************************************************\n* Alloy Integrations\n*****************************************************************/\nremote.kubernetes.configmap \"integrations\" {\n\tnamespace = \"monitoring-system\"\n\tname = \"alloy-integrations\"\n}\n\n// Memcached Integrations\nimport.string \"memcached\" {\n\tcontent = remote.kubernetes.configmap.integrations.data[\"memcached.alloy\"]\n}\n\nmemcached.memcached_metrics_scrape - \"instance\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\n\tnamespace - = \"monitoring-system\"\n\tname = remote.kubernetes.configmap.integrations.data[\"MEMCACHED_K8S_SECRET_NAME\"]\n}\n\n// - // Redis Integrations\n// import.string \"redis\" {\n// \tcontent = remote.kubernetes.configmap.integrations.data[\"redis.alloy\"]\n// - }\n\n// redis.redis_exporter_metrics_scrape \"instance\" {\n// \tforward_to = - [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\n// \tnamespace = \"monitoring-system\"\n// - \tname = remote.kubernetes.configmap.integrations.data[\"REDIS_K8S_SECRET_NAME\"]\n// - }\n\n// // Mysql Integrations\n// import.string \"mysql\" {\n// \tcontent = remote.kubernetes.configmap.integrations.data[\"mysql.alloy\"]\n// - }\n\n// mysql.mysql_metrics_scrape \"instance\" {\n// \tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\n// - \tnamespace = \"monitoring-system\"\n// \tname = remote.kubernetes.configmap.integrations.data[\"MYSQL_K8S_SECRET_NAME\"]\n// - }\n" + \"instance\" {\n\tnamespace = \"monitoring-system\"\n\tname = remote.kubernetes.configmap.integrations.data[\"MEMCACHED_K8S_SECRET_NAME\"]\n\n\tforward_to + = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-72927d7k99 + name: alloy-config-fmt85fgk68 namespace: monitoring-system --- apiVersion: v1 @@ -489,28 +485,25 @@ apiVersion: v1 data: annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: Scrapes targets for logs based on kubernetes Pod annotations\n\n Annotations:\n - \ logs.grafana.com/ingest: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare + \ logs.grafana.com/scrape: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this - can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\t// arguments - for kubernetes discovery\n\targument \"namespaces\" {\n\t\tcomment = \"The namespaces - to look for targets in (default: [\\\"kube-system\\\"] is all namespaces)\"\n\t\toptional - = true\n\t}\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\targument + \"cluster\" { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix to use (default: logs.grafana.com)\"\n\t\tdefault = \"logs.grafana.com\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional = true\n\t\tcomment \ = \"The logic is used to transform the annotation argument into a valid label name by removing unsupported characters.\"\n\t\tdefault = replace(replace(replace(coalesce(argument.annotation_prefix.value, \"logs.grafana.com\"), \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\t// - find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\n\t\tnamespaces - {\n\t\t\tnames = coalesce(argument.namespaces.value, [])\n\t\t}\n\t}\n\n\t// filter - logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" + find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\t}\n\n\t// + filter logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" {\n\t\ttargets = discovery.kubernetes.annotation_logs.targets\n\n\t\t// allow pods to declare their logs to be ingested or not, the default is true\n\t\t// - \ i.e. logs.grafana.com/ingest: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + \ i.e. logs.grafana.com/scrape: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\n\t\t// allow pods to declare what tenant their logs should be written to, the following annotation @@ -520,6 +513,8 @@ data: + argument.tenant.value + \")$\"\n\t\t}\n\n\t\t// set the instance label as the name of the worker node the pod is on\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_node_name\"]\n\t\t\ttarget_label = \"instance\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\t// set the pod label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\t// @@ -584,12 +579,11 @@ data: ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\toptional - = true\n\t}\n\n\targument \"cluster\" {\n\t\toptional = true\n\t}\n\n\tloki.source.kubernetes_events - \"cluster_events\" {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format + = true\n\t}\n\n\targument \"cluster\" { }\n\n\tloki.source.kubernetes_events \"cluster_events\" + {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format = \"logfmt\"\n\t\tforward_to = [loki.process.logs_service.receiver]\n\t}\n\n\tloki.process \"logs_service\" {\n\t\tstage.static_labels {\n\t\t\tvalues = {\n\t\t\t\tcluster - = coalesce(argument.cluster.value, \"k3d\"),\n\t\t\t}\n\t\t}\n\t\tforward_to = - argument.forward_to.value\n\t}\n}\n" + = argument.cluster.value,\n\t\t\t}\n\t\t}\n\t\tforward_to = argument.forward_to.value\n\t}\n}\n" keep-labels.alloy: "/*\nModule Components: keep_labels\nDescription: Pre-defined set of labels to keep, this stage should always be in-place as the previous relabeing\n \ stages make every pod label and annotation a label in the pipeline, @@ -611,46 +605,392 @@ data: = argument.keep_labels.value\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* EXPORTS\n\t*****************************************************************/\n\texport \"receiver\" {\n\t\tvalue = loki.process.keep_labels.receiver\n\t}\n}\n" + rules-to-loki.alloy: "/*\nModule Components: rules_to_loki\nDescription: Auto discovers + PrometheusRule Kubernetes resources and loads them into a Loki instance.\n*/\n\ndeclare + \"rules_to_loki\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + fake)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Loki\n\t********************************************/\n\tloki.rules.kubernetes + \"rules_to_loki\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:3100\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_loki= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\trule_selector {\n\t\t\tmatch_labels = {\n\t\t\t\tauto_rules_to_loki + = \"true\",\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k namespace: monitoring-system --- apiVersion: v1 data: + annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: + Scrapes targets for metrics based on kubernetes annotations\n\nNote: Every argument + except for \"forward_to\" is optional, and does have a defined default value. + \ However, the values for these\n arguments are not defined using the default + = \" ... \" argument syntax, but rather using the coalesce(argument.value, \" + ... \").\n This is because if the argument passed in from another consuming + module is set to null, the default = \" ... \" syntax will\n does not override + the value passed in, where coalesce() will return the first non-null value.\n\nKubernetes + Annotation Auto-Scraping\n------------------------------------------------------------------------------------------------------------------------------------\nThis + module is meant to be used to automatically scrape targets based on a certain + role and set of annotations. This module can be consumed\nmultiple times with + different roles. The supported roles are:\n\n - pod\n - service\n - endpoints\n\nTypically, + if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality + you would use role=\"endpoints\", if\nmimicking the behavior of the PodMonitor + functionality you would use role=\"pod\". It is important to understand that + with endpoints,\nthe target is typically going to be a pod, and whatever annotations + that are set on the service will automatically be propagated to the\nendpoints. + \ This is why the role \"endpoints\" is used, because it will scrape the pod, + but also consider the service annotations. Using\nrole=\"endpoints\", which scrape + each endpoint associated to the service. If role=\"service\" is used, it will + only scrape the service, only\nhitting one of the endpoints associated to the + service.\n\nThis is where you must consider your scraping strategy, for example + if you scrape a service like \"kube-state-metrics\" using\nrole=\"endpoints\" + you should only have a single replica of the kube-state-metrics pod, if you have + multiple replicas, you should use\nrole=\"service\" or a separate non-annotation + job completely. Scraping a service instead of endpoints, is typically a rare + use case, but\nit is supported.\n\nThere are other considerations for using annotation + based scraping as well, which is metric relabeling rules that happen post scrape. + \ If\nyou have a target that you want to apply a bunch of relabelings to or a + very large metrics response payload, performance wise it will be\nbetter to have + a separate job for that target, rather than using use annotations. As every targert + will go through the ssame relabeling.\nTypical deployment strategies/options would + be:\n\nOption #1 (recommended):\n - Annotation Scraping for role=\"endpoints\"\n + \ - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, + etc.) or large metric payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, + kube-apiserver, kube-scheduler, etc.)\n\nOption #2:\n - Annotation Scraping for + role=\"pod\"\n - Annotation Scraping for role=\"service\" (i.e. kube-state-metrics, + node-exporter, etc.)\n - Separate Jobs for specific use cases or large metric + payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, + kube-scheduler, etc.)\n\nAt no point should you use role=\"endpoints\" and role=\"pod\" + together, as this will result in duplicate targets being scraped, thus\ngenerating + duplicate metrics. If you want to scrape both the pod and the service, use Option + #2.\n\nEach port attached to an service/pod/endpoint is an eligible target, oftentimes + it will have multiple ports.\nThere may be instances when you want to scrape all + ports or some ports and not others. To support this\nthe following annotations + are available:\n\n metrics.grafana.com/scrape: true\n\nthe default scraping scheme + is http, this can be specified as a single value which would override, the schema + being used for all\nports attached to the target:\n\n metrics.grafana.com/scheme: + https\n\nthe default path to scrape is /metrics, this can be specified as a single + value which would override, the scrape path being used\nfor all ports attached + to the target:\n\n metrics.grafana.com/path: /metrics/some_path\n\nthe default + port to scrape is the target port, this can be specified as a single value which + would override the scrape port being\nused for all ports attached to the target, + note that even if aan target had multiple targets, the relabel_config targets + are\ndeduped before scraping:\n\n metrics.grafana.com/port: 8080\n\nthe default + interval to scrape is 1m, this can be specified as a single value which would + override, the scrape interval being used\nfor all ports attached to the target:\n\n + \ metrics.grafana.com/interval: 5m\n\nthe default timeout for scraping is 10s, + this can be specified as a single value which would override, the scrape interval + being\nused for all ports attached to the target:\n\n metrics.grafana.com/timeout: + 30s\n\nthe default job is namespace/{{ service name }} or namespace/{{ controller_name + }} depending on the role, there may be instances\nin which a different job name + is required because of a set of dashboards, rules, etc. to support this there + is a job annotation\nwhich will override the default value:\n\n metrics.grafana.com/job: + integrations/kubernetes/kube-state-metrics\n*/\n\ndeclare \"annotations_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment + \ = \"The tenant to filter metrics to. This does not have to be the tenantId, + this is the value to look for in the metrics.agent.grafana.com/tenant annotation, + and this can be a regex.\"\n\t\toptional = true\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + to use (default: metrics.grafana.com)\"\n\t\tdefault = \"metrics.grafana.com\"\n\t\toptional + = true\n\t}\n\n\targument \"role\" {\n\t\tcomment = \"The role to use when looking + for targets to scrape via annotations, can be: endpoints, service, pod (default: + endpoints)\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional + = true\n\t\tcomment = \"The logic is used to transform the annotation argument + into a valid label name by removing unsupported characters.\"\n\t\tdefault = + replace(replace(replace(coalesce(argument.annotation_prefix.value, \"metrics.grafana.com\"), + \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\targument \"__pod_role\" + {\n\t\tcomment = \"Most annotation targets service or pod that is all you want, + however if the role is endpoints you want the pod\"\n\t\toptional = true\n\t\tdefault + \ = replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"pod\")\n\t}\n\n\targument + \"__service_role\" {\n\t\tcomment = \"Most annotation targets service or pod + that is all you want, however if the role is endpoints you we also want to consider + service annotations\"\n\t\toptional = true\n\t\tdefault = replace(coalesce(argument.role.value, + \"endpoints\"), \"endpoints\", \"service\")\n\t}\n\n\targument \"scrape_port_named_metrics\" + {\n\t\tcomment = \"Whether or not to automatically scrape endpoints that have + a port with 'metrics' in the name\"\n\t\toptional = true\n\t\tdefault = false\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: (.+))\"\n\t\toptional + = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment = \"A regex of metrics + to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" + {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + Targets From Docker Discovery\n\t*****************************************************************/\n\tdiscovery.kubernetes + \"annotation_metrics\" {\n\t\trole = coalesce(argument.role.value, \"endpoints\")\n\n\t\tselectors + {\n\t\t\trole = coalesce(argument.role.value, \"endpoints\")\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t*****************************************************************/\n\t// + filter metrics by kubernetes annotations\n\tdiscovery.relabel \"annotation_metrics_filter\" + {\n\t\ttargets = discovery.kubernetes.annotation_metrics.targets\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Targets to Keep or Drop\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare their metrics scraped or not\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/scrape: false\n\t\t//\n\t\t// the label prometheus.io/service-monitor: + \"false\" is a common label for headless services, when performing endpoint\n\t\t// + service discovery, if there is both a load-balanced service and headless service, + this can result in duplicate\n\t\t// scrapes if the name of the service is attached + as a label. any targets with this label or annotation set should be dropped\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"false\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, + \"endpoints\") + \"_label_prometheus_io_service_monitor\",\n\t\t\t]\n\t\t\tseparator + = \";\"\n\t\t\t// only allow empty or true, otherwise defaults to false\n\t\t\tregex + \ = \"^(?:;*)?(true)(;|true)*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\t// add a __tmp_scrape_port_named_metrics from + the argument.scrape_port_named_metrics\n\t\trule {\n\t\t\treplacement = format(\"%t\", + argument.scrape_port_named_metrics.value)\n\t\t\ttarget_label = \"__tmp_scrape_port_named_metrics\"\n\t\t}\n\n\t\t// + only keep targets that have scrape: true or \"metrics\" in the port name if the + argument scrape_port_named_metrics\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__tmp_scrape\",\n\t\t\t\t\"__tmp_scrape_port_named_metrics\",\n\t\t\t\t// + endpoints is the role and most meta labels started with \"endpoints\", however + the port name is an exception and starts with \"endpoint\"\n\t\t\t\t\"__meta_kubernetes_\" + + replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"endpoint\") + + \"_port_name\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(true;.*|(|true);true;(.*metrics.*))$\"\n\t\t}\n\n\t\t// + only keep targets where the pod is running or the pod_phase is empty and is not + an init container. This will only exist for role=\"pod\" or\n\t\t// potentially + role=\"endpoints\", if it is a service the value is empty and thus allowed to + pass, if it is an endpoint but not associated to a\n\t\t// pod but rather a static + IP or hostname, that could be outside of kubernetes allow endpoints to declare + what tenant their metrics should be\n\t\t// written to\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex + \ = \"^(?i)(Running|)$\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__meta_kubernetes_pod_ready\"]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\t\t// + if the container is an init container, drop it\n\t\trule {\n\t\t\taction = + \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_container_init\"]\n\t\t\tregex + \ = \"^(true)$\"\n\t\t}\n\n\t\t// allow resources to declare their metrics + the tenant their metrics should be sent to,\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/tenant: primary\n\t\t//\n\t\t// Note: This does not necessarily + have to be the actual tenantId, it can be a friendly name as well that is simply + used\n\t\t// to determine if the metrics should be gathered for the current + tenant\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_tenant\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_tenant\",\n\t\t\t]\n\t\t\tregex + = \"^(\" + coalesce(argument.tenant.value, \".*\") + \")$\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Scrape Metadata i.e. path, port, interval etc.\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare the protocol to use when collecting metrics, the default + value is \"http\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/scheme: + http\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"http\"\n\t\t\ttarget_label + = \"__scheme__\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scheme\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scheme\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?(https?).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scheme__\"\n\t\t}\n\n\t\t// allow resources + to declare the port to use when collecting metrics, the default value is the discovered + port from\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/port: 9090\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__address__\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_port\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_port\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^:]+)(?::\\\\d+)?;(\\\\d+)$\"\n\t\t\treplacement + \ = \"$1:$2\"\n\t\t\ttarget_label = \"__address__\"\n\t\t}\n\n\t\t// allow resources + to declare their the path to use when collecting their metrics, the default value + is \"/metrics\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/path: + /metrics/foo\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_path\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_path\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__metrics_path__\"\n\t\t}\n\n\t\t// allow resources + to declare how often their metrics should be collected, the default value is 1m,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/interval: 5m\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_interval\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_interval\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\t// allow + resources to declare the timeout of the scrape request, the default value is 10s,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/timeout: 30s\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_timeout\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_timeout\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Common Labels\n\t\t****************************************************************************************************************/\n\t\t// + set a source label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = \"kubernetes\"\n\t\t\ttarget_label = \"source\"\n\t\t}\n\n\t\t// set the cluster + label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = argument.cluster.value\n\t\t\ttarget_label + = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label + \ = \"namespace\"\n\t\t}\n\n\t\t// set the target name label i.e. service name, + pod name, etc.\n\t\t// if the role is endpoints, the first valued field is used + which would be __meta_kubernetes_pod_name, if the pod name is empty\n\t\t// then + the endpoint name would be used\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + argument.__pod_role.value + \"_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = + \"$1\"\n\t\t\ttarget_label = argument.__pod_role.value\n\t\t}\n\n\t\t// set a + default job label to be the namespace/pod_controller_name or namespace/service_name\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t\targument.__pod_role.value,\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^;]+)(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// if the controller + is a ReplicaSet, drop the hash from the end of the ReplicaSet\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_type\",\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:ReplicaSet);([^;]+);([^;]+)-.+$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// allow resources to + declare their the job label value to use when collecting their metrics, the default + value is \"\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/job: + integrations/kubernetes/cadvisor\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_job\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_job\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// set the app name if + specified as metadata labels \"app:\" or \"app.kubernetes.io/name:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"app\"\n\t\t}\n\n\t\t// + set the app component if specified as metadata labels \"component:\" or \"app.kubernetes.io/component:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_component\",\n\t\t\t]\n\t\t\tregex + \ = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"component\"\n\t\t}\n\n\t\t// set the version if specified as metadata labels + \"version:\" or \"app.kubernetes.io/version:\" or \"app_version:\"\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_version\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"version\"\n\t\t}\n\n\t\t// + set a workload label if the resource is a pod\n\t\t// example: grafana-agent-68nv9 + becomes DaemonSet/grafana-agent\n\t\trule {\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_kind\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"(.+);(.+)\"\n\t\t\treplacement = \"$1/$2\"\n\t\t\ttarget_label + = \"workload\"\n\t\t}\n\t\t// remove the hash from the ReplicaSet\n\t\trule {\n\t\t\tsource_labels + = [\"workload\"]\n\t\t\tregex = \"(ReplicaSet/.+)-.+\"\n\t\t\ttarget_label + \ = \"workload\"\n\t\t}\n\t}\n\n\t// only keep http targets\n\tdiscovery.relabel + \"http_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"http\"\n\t\t}\n\t}\n\n\t// only keep https targets\n\tdiscovery.relabel + \"https_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"https\"\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Scrape Labels Targets\n\t*****************************************************************/\n\t// + scrape http only targtetsa\n\tprometheus.scrape \"http_annotations\" {\n\t\ttargets + = discovery.relabel.http_annotations.output\n\n\t\tjob_name = \"annotation-metrics-http\"\n\t\tscheme + \ = \"http\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t// + scrape https only targtets\n\tprometheus.scrape \"https_annotations\" {\n\t\ttargets + = discovery.relabel.https_annotations.output\n\n\t\tjob_name = \"annotation-metrics-https\"\n\t\tscheme + \ = \"https\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\tbearer_token_file + = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n\n\t\ttls_config {\n\t\t\tca_file + \ = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n\t\t\tinsecure_skip_verify + = false\n\t\t\tserver_name = \"kubernetes\"\n\t\t}\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"annotations\" {\n\t\tforward_to = argument.forward_to.value\n\t\t// keep only + metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(.+)\")\n\t\t}\n\n\t\t// drop metrics that match the drop_metrics regex\n\t\trule + {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex + \ = coalesce(argument.drop_metrics.value, \"\")\n\t\t}\n\t}\n}\n" podmonitors-scrape.alloy: "/*\nModule Components: podmonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.podmonitors\n*/\n\ndeclare \"podmonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape PodMonitors\n\t*****************************************************************/\n\tprometheus.operator.podmonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.podmonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\t// selector {\n\t\t// \tmatch_expression {\n\t\t// \t\tkey = \"team\"\n\t\t// \t\toperator = \"In\"\n\t\t// \t\tvalues - \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n}\n" + \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"podmonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set the + cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" + rules-to-mimir.alloy: "/*\nModule Components: rules_to_mimir\nDescription: Auto + discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance.\n*/\n\ndeclare + \"rules_to_mimir\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + anonymous)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Mimir\n\t********************************************/\n\tmimir.rules.kubernetes + \"rules_to_mimir\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:8080\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\t// rule_selector {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= + \"true\",\n\t\t// \t}\n\t\t// }\n\t}\n}\n" servicemonitors-scrape.alloy: "/*\nModule Components: servicemonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.servicemonitors\n*/\n\ndeclare \"servicemonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape ServiceMonitors\n\t*****************************************************************/\n\tprometheus.operator.servicemonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.servicemonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering - {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n" + {\n\t\t\tenabled = true\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"servicemonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set + the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f namespace: monitoring-system --- apiVersion: v1 @@ -660,19 +1000,21 @@ data: \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(ProfilessReceiver) where collected - logs should be forwarded to\"\n\t}\n\n\tdiscovery.kubernetes \"pyroscope_kubernetes\" - {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape config allows to define - annotations based scraping.\n\t//\n\t// For example the following annotations:\n\t//\n\t// - ```\n\t// profiles.grafana.com/memory.scrape: \"true\"\n\t// profiles.grafana.com/memory.port: - \"8080\"\n\t// profiles.grafana.com/cpu.scrape: \"true\"\n\t// profiles.grafana.com/cpu.port: - \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: \"true\"\n\t// profiles.grafana.com/goroutine.port: - \"8080\"\n\t// ```\n\t//\n\t// will scrape the `memory`, `cpu` and `goroutine` - profiles from the `8080` port of the pod.\n\t//\n\t// For more information see - https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel + logs should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\tdiscovery.kubernetes + \"pyroscope_kubernetes\" {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape + config allows to define annotations based scraping.\n\t//\n\t// For example the + following annotations:\n\t//\n\t// ```\n\t// profiles.grafana.com/memory.scrape: + \"true\"\n\t// profiles.grafana.com/memory.port: \"8080\"\n\t// profiles.grafana.com/cpu.scrape: + \"true\"\n\t// profiles.grafana.com/cpu.port: \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: + \"true\"\n\t// profiles.grafana.com/goroutine.port: \"8080\"\n\t// ```\n\t//\n\t// + will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port + of the pod.\n\t//\n\t// For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel \"kubernetes_pods\" {\n\t\ttargets = concat(discovery.kubernetes.pyroscope_kubernetes.targets)\n\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex \ = \"Pending|Succeeded|Failed|Completed\"\n\t\t}\n\n\t\trule {\n\t\t\taction - = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\trule + = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\trule @@ -891,7 +1233,7 @@ data: {\n\t\t\t\tenabled = true\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 namespace: monitoring-system --- apiVersion: v1 @@ -901,40 +1243,58 @@ data: Transformation\ndeclare \"process_and_transform\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"traces_forward_to\" {\n\t\tcomment = \"Must be a list(TracesReceiver) where - collected traces should be forwarded to\"\n\t}\n\n\targument \"logs_forward_to\" - {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be - forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment = \"Must - be a list(MetricsReceiver) where collected metrics should be forwarded to\"\n\t}\n\n\targument - \"cluster\" {\n\t\toptional = true\n\t\tdefault = \"k3d-k3s-codelab\"\n\t}\n\n\targument - \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4318\"\n\t}\n\n\targument - \"otlp_grpc_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* + collected traces should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\targument + \"logs_forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected + logs should be forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment + = \"Must be a list(MetricsReceiver) where collected metrics should be forwarded + to\"\n\t}\n\n\targument \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault + \ = \"0.0.0.0:4318\"\n\t}\n\n\targument \"otlp_grpc_endpoint\" {\n\t\toptional + = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* Jaeger for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.jaeger \"default\" {\n\t\tprotocols {\n\t\t\tgrpc {\n\t\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t\t}\n\n\t\t\tthrift_http {\n\t\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t\t}\n\n\t\t\tthrift_binary {\n\t\t\t\tendpoint = \"0.0.0.0:6832\"\n\t\t\t}\n\n\t\t\tthrift_compact {\n\t\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [otelcol.processor.resourcedetection.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + {\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* Otelcol for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.otlp - \"default\" {\n\t\tgrpc {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp - {\n\t\t\tendpoint = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics - = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection - \"default\" {\n\t\tdetectors = [\"env\"]\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces - = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes + \"default\" {\n\t\tdebug_metrics {\n\t\t\tdisable_high_cardinality_metrics = true\n\t\t}\n\n\t\tgrpc + {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp {\n\t\t\tendpoint + = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.resourcedetection.default.input]\n\t\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection + \"default\" {\n\t\tdetectors = [\"env\", \"system\"]\n\n\t\tsystem {\n\t\t\thostname_sources + = [\"os\"]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\n\t\t\tlogs + \ = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform + \"add_metric_datapoint_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements + {\n\t\t\tcontext = \"datapoint\"\n\t\t\tstatements = [\n\t\t\t\t\"set(attributes[\\\"deployment.environment\\\"], + resource.attributes[\\\"deployment.environment\\\"])\",\n\t\t\t\t\"set(attributes[\\\"service.version\\\"], + resource.attributes[\\\"service.version\\\"])\",\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes \"default\" {\n\t\textract {\n\t\t\tmetadata = [\n\t\t\t\t\"k8s.namespace.name\",\n\t\t\t\t\"k8s.pod.name\",\n\t\t\t\t\"k8s.deployment.name\",\n\t\t\t\t\"k8s.statefulset.name\",\n\t\t\t\t\"k8s.daemonset.name\",\n\t\t\t\t\"k8s.cronjob.name\",\n\t\t\t\t\"k8s.job.name\",\n\t\t\t\t\"k8s.node.name\",\n\t\t\t\t\"k8s.pod.uid\",\n\t\t\t\t\"k8s.pod.start_time\",\n\t\t\t]\n\t\t}\n\n\t\tpod_association {\n\t\t\tsource {\n\t\t\t\tfrom = \"connection\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t\ttraces - = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform - \"add_resource_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tlog_statements + {\n\t\t\tmetrics = [otelcol.processor.transform.default.input]\n\t\t\tlogs = + [otelcol.processor.transform.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.transform.default.input,\n\t\t\t\totelcol.connector.host_info.default.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.connector.host_info + \"default\" {\n\t\thost_identifiers = [\"k8s.node.name\"]\n\n\t\toutput {\n\t\t\tmetrics + = [otelcol.processor.batch.host_info_batch.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + \"host_info_batch\" {\n\t\toutput {\n\t\t\tmetrics = [otelcol.exporter.prometheus.host_info_metrics.input]\n\t\t}\n\t}\n\n\totelcol.exporter.prometheus + \"host_info_metrics\" {\n\t\tadd_metric_suffixes = false\n\t\tforward_to = + argument.metrics_forward_to.value\n\t}\n\n\totelcol.processor.transform \"default\" + {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements {\n\t\t\tcontext = + \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\tlog_statements {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])`,\n\t\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t\t`set(attributes[\"loki.resource.labels\"], \"pod, namespace, cluster, job\")`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements - {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], - \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter - \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.batch.default.input]\n\t\t\ttraces - = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements + {\n\t\t\tcontext = \"span\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces + \ = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter + \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\ttraces {\n\t\t\tspan = [\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/live\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] == \\\"/healthy\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/ready\\\"\",\n\t\t\t]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs + \ = [otelcol.processor.batch.default.input]\n\t\t\ttraces = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch \"default\" {\n\t\tsend_batch_size = 16384\n\t\tsend_batch_max_size = 0\n\t\ttimeout \ = \"5s\"\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\t\tlogs \ = [otelcol.processor.memory_limiter.default.input]\n\t\t\ttraces = [otelcol.processor.memory_limiter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.memory_limiter @@ -970,7 +1330,7 @@ data: \"alloy_traces_input\" {\n\t\tvalue = otelcol.processor.batch.default.input\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 namespace: monitoring-system --- apiVersion: v1 @@ -3046,19 +3406,19 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-72927d7k99 + name: alloy-config-fmt85fgk68 name: config - configMap: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f name: modules-kubernetes-metrics - configMap: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k name: modules-kubernetes-logs - configMap: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 name: modules-kubernetes-traces - configMap: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 name: modules-kubernetes-profiles --- apiVersion: policy/v1 diff --git a/kubernetes/microservices-mode/logs/logs.alloy b/kubernetes/microservices-mode/logs/logs.alloy index 514a59e7..f7d59991 100644 --- a/kubernetes/microservices-mode/logs/logs.alloy +++ b/kubernetes/microservices-mode/logs/logs.alloy @@ -25,9 +25,16 @@ import.file "logs" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/logs" } +logs.kubernetes_cluster_events "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + + forward_to = [logs.keep_labels.kubernetes.receiver] +} + logs.annotations_scrape "kubernetes" { - annotation_prefix = "logs.grafana.com" - forward_to = [logs.keep_labels.kubernetes.receiver] + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + + forward_to = [logs.keep_labels.kubernetes.receiver] } logs.keep_labels "kubernetes" { @@ -41,11 +48,17 @@ import.file "metrics" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/metrics" } -metrics.podmonitors_scrape "kubernetes" { +metrics.annotations_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } metrics.servicemonitors_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } @@ -63,32 +76,8 @@ import.string "memcached" { } memcached.memcached_metrics_scrape "instance" { - forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] - namespace = "monitoring-system" name = remote.kubernetes.configmap.integrations.data["MEMCACHED_K8S_SECRET_NAME"] -} - -// // Redis Integrations -// import.string "redis" { -// content = remote.kubernetes.configmap.integrations.data["redis.alloy"] -// } - -// redis.redis_exporter_metrics_scrape "instance" { -// forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] - -// namespace = "monitoring-system" -// name = remote.kubernetes.configmap.integrations.data["REDIS_K8S_SECRET_NAME"] -// } -// // Mysql Integrations -// import.string "mysql" { -// content = remote.kubernetes.configmap.integrations.data["mysql.alloy"] -// } - -// mysql.mysql_metrics_scrape "instance" { -// forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] - -// namespace = "monitoring-system" -// name = remote.kubernetes.configmap.integrations.data["MYSQL_K8S_SECRET_NAME"] -// } + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] +} diff --git a/kubernetes/microservices-mode/metrics/k8s-all-in-one.yaml b/kubernetes/microservices-mode/metrics/k8s-all-in-one.yaml index 10998a43..cb622b54 100644 --- a/kubernetes/microservices-mode/metrics/k8s-all-in-one.yaml +++ b/kubernetes/microservices-mode/metrics/k8s-all-in-one.yaml @@ -166,43 +166,38 @@ data: = coalesce(env(\"SELF_HOSTED_METRICS_ENDPOINT_URL\"), \"http://nginx.gateway.svc:8080/api/v1/push\")\n}\n\n/********************************************\n * Metrics\n ********************************************/\nimport.file \"metrics\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/metrics\"\n}\n\nmetrics.podmonitors_scrape \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape - \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\n// - metrics.annotations_scrape \"kubernetes\" {\n// \tlabel_prefix = \"metrics.grafana.com\"\n// - \tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n// - \tscrape_interval = \"15s\"\n// }\n" + + \"/kubernetes/metrics\"\n}\n\nmetrics.annotations_scrape \"kubernetes\" {\n\tforward_to + = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\n\tcluster = + coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval = \"15s\"\n}\n\nmetrics.servicemonitors_scrape + \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-55dd47md5f + name: alloy-config-dkdkkgdd64 namespace: monitoring-system --- apiVersion: v1 data: annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: Scrapes targets for logs based on kubernetes Pod annotations\n\n Annotations:\n - \ logs.grafana.com/ingest: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare + \ logs.grafana.com/scrape: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this - can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\t// arguments - for kubernetes discovery\n\targument \"namespaces\" {\n\t\tcomment = \"The namespaces - to look for targets in (default: [\\\"kube-system\\\"] is all namespaces)\"\n\t\toptional - = true\n\t}\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\targument + \"cluster\" { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix to use (default: logs.grafana.com)\"\n\t\tdefault = \"logs.grafana.com\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional = true\n\t\tcomment \ = \"The logic is used to transform the annotation argument into a valid label name by removing unsupported characters.\"\n\t\tdefault = replace(replace(replace(coalesce(argument.annotation_prefix.value, \"logs.grafana.com\"), \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\t// - find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\n\t\tnamespaces - {\n\t\t\tnames = coalesce(argument.namespaces.value, [])\n\t\t}\n\t}\n\n\t// filter - logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" + find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\t}\n\n\t// + filter logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" {\n\t\ttargets = discovery.kubernetes.annotation_logs.targets\n\n\t\t// allow pods to declare their logs to be ingested or not, the default is true\n\t\t// - \ i.e. logs.grafana.com/ingest: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + \ i.e. logs.grafana.com/scrape: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\n\t\t// allow pods to declare what tenant their logs should be written to, the following annotation @@ -212,6 +207,8 @@ data: + argument.tenant.value + \")$\"\n\t\t}\n\n\t\t// set the instance label as the name of the worker node the pod is on\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_node_name\"]\n\t\t\ttarget_label = \"instance\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\t// set the pod label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\t// @@ -276,12 +273,11 @@ data: ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\toptional - = true\n\t}\n\n\targument \"cluster\" {\n\t\toptional = true\n\t}\n\n\tloki.source.kubernetes_events - \"cluster_events\" {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format + = true\n\t}\n\n\targument \"cluster\" { }\n\n\tloki.source.kubernetes_events \"cluster_events\" + {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format = \"logfmt\"\n\t\tforward_to = [loki.process.logs_service.receiver]\n\t}\n\n\tloki.process \"logs_service\" {\n\t\tstage.static_labels {\n\t\t\tvalues = {\n\t\t\t\tcluster - = coalesce(argument.cluster.value, \"k3d\"),\n\t\t\t}\n\t\t}\n\t\tforward_to = - argument.forward_to.value\n\t}\n}\n" + = argument.cluster.value,\n\t\t\t}\n\t\t}\n\t\tforward_to = argument.forward_to.value\n\t}\n}\n" keep-labels.alloy: "/*\nModule Components: keep_labels\nDescription: Pre-defined set of labels to keep, this stage should always be in-place as the previous relabeing\n \ stages make every pod label and annotation a label in the pipeline, @@ -303,46 +299,392 @@ data: = argument.keep_labels.value\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* EXPORTS\n\t*****************************************************************/\n\texport \"receiver\" {\n\t\tvalue = loki.process.keep_labels.receiver\n\t}\n}\n" + rules-to-loki.alloy: "/*\nModule Components: rules_to_loki\nDescription: Auto discovers + PrometheusRule Kubernetes resources and loads them into a Loki instance.\n*/\n\ndeclare + \"rules_to_loki\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + fake)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Loki\n\t********************************************/\n\tloki.rules.kubernetes + \"rules_to_loki\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:3100\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_loki= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\trule_selector {\n\t\t\tmatch_labels = {\n\t\t\t\tauto_rules_to_loki + = \"true\",\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k namespace: monitoring-system --- apiVersion: v1 data: + annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: + Scrapes targets for metrics based on kubernetes annotations\n\nNote: Every argument + except for \"forward_to\" is optional, and does have a defined default value. + \ However, the values for these\n arguments are not defined using the default + = \" ... \" argument syntax, but rather using the coalesce(argument.value, \" + ... \").\n This is because if the argument passed in from another consuming + module is set to null, the default = \" ... \" syntax will\n does not override + the value passed in, where coalesce() will return the first non-null value.\n\nKubernetes + Annotation Auto-Scraping\n------------------------------------------------------------------------------------------------------------------------------------\nThis + module is meant to be used to automatically scrape targets based on a certain + role and set of annotations. This module can be consumed\nmultiple times with + different roles. The supported roles are:\n\n - pod\n - service\n - endpoints\n\nTypically, + if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality + you would use role=\"endpoints\", if\nmimicking the behavior of the PodMonitor + functionality you would use role=\"pod\". It is important to understand that + with endpoints,\nthe target is typically going to be a pod, and whatever annotations + that are set on the service will automatically be propagated to the\nendpoints. + \ This is why the role \"endpoints\" is used, because it will scrape the pod, + but also consider the service annotations. Using\nrole=\"endpoints\", which scrape + each endpoint associated to the service. If role=\"service\" is used, it will + only scrape the service, only\nhitting one of the endpoints associated to the + service.\n\nThis is where you must consider your scraping strategy, for example + if you scrape a service like \"kube-state-metrics\" using\nrole=\"endpoints\" + you should only have a single replica of the kube-state-metrics pod, if you have + multiple replicas, you should use\nrole=\"service\" or a separate non-annotation + job completely. Scraping a service instead of endpoints, is typically a rare + use case, but\nit is supported.\n\nThere are other considerations for using annotation + based scraping as well, which is metric relabeling rules that happen post scrape. + \ If\nyou have a target that you want to apply a bunch of relabelings to or a + very large metrics response payload, performance wise it will be\nbetter to have + a separate job for that target, rather than using use annotations. As every targert + will go through the ssame relabeling.\nTypical deployment strategies/options would + be:\n\nOption #1 (recommended):\n - Annotation Scraping for role=\"endpoints\"\n + \ - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, + etc.) or large metric payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, + kube-apiserver, kube-scheduler, etc.)\n\nOption #2:\n - Annotation Scraping for + role=\"pod\"\n - Annotation Scraping for role=\"service\" (i.e. kube-state-metrics, + node-exporter, etc.)\n - Separate Jobs for specific use cases or large metric + payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, + kube-scheduler, etc.)\n\nAt no point should you use role=\"endpoints\" and role=\"pod\" + together, as this will result in duplicate targets being scraped, thus\ngenerating + duplicate metrics. If you want to scrape both the pod and the service, use Option + #2.\n\nEach port attached to an service/pod/endpoint is an eligible target, oftentimes + it will have multiple ports.\nThere may be instances when you want to scrape all + ports or some ports and not others. To support this\nthe following annotations + are available:\n\n metrics.grafana.com/scrape: true\n\nthe default scraping scheme + is http, this can be specified as a single value which would override, the schema + being used for all\nports attached to the target:\n\n metrics.grafana.com/scheme: + https\n\nthe default path to scrape is /metrics, this can be specified as a single + value which would override, the scrape path being used\nfor all ports attached + to the target:\n\n metrics.grafana.com/path: /metrics/some_path\n\nthe default + port to scrape is the target port, this can be specified as a single value which + would override the scrape port being\nused for all ports attached to the target, + note that even if aan target had multiple targets, the relabel_config targets + are\ndeduped before scraping:\n\n metrics.grafana.com/port: 8080\n\nthe default + interval to scrape is 1m, this can be specified as a single value which would + override, the scrape interval being used\nfor all ports attached to the target:\n\n + \ metrics.grafana.com/interval: 5m\n\nthe default timeout for scraping is 10s, + this can be specified as a single value which would override, the scrape interval + being\nused for all ports attached to the target:\n\n metrics.grafana.com/timeout: + 30s\n\nthe default job is namespace/{{ service name }} or namespace/{{ controller_name + }} depending on the role, there may be instances\nin which a different job name + is required because of a set of dashboards, rules, etc. to support this there + is a job annotation\nwhich will override the default value:\n\n metrics.grafana.com/job: + integrations/kubernetes/kube-state-metrics\n*/\n\ndeclare \"annotations_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment + \ = \"The tenant to filter metrics to. This does not have to be the tenantId, + this is the value to look for in the metrics.agent.grafana.com/tenant annotation, + and this can be a regex.\"\n\t\toptional = true\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + to use (default: metrics.grafana.com)\"\n\t\tdefault = \"metrics.grafana.com\"\n\t\toptional + = true\n\t}\n\n\targument \"role\" {\n\t\tcomment = \"The role to use when looking + for targets to scrape via annotations, can be: endpoints, service, pod (default: + endpoints)\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional + = true\n\t\tcomment = \"The logic is used to transform the annotation argument + into a valid label name by removing unsupported characters.\"\n\t\tdefault = + replace(replace(replace(coalesce(argument.annotation_prefix.value, \"metrics.grafana.com\"), + \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\targument \"__pod_role\" + {\n\t\tcomment = \"Most annotation targets service or pod that is all you want, + however if the role is endpoints you want the pod\"\n\t\toptional = true\n\t\tdefault + \ = replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"pod\")\n\t}\n\n\targument + \"__service_role\" {\n\t\tcomment = \"Most annotation targets service or pod + that is all you want, however if the role is endpoints you we also want to consider + service annotations\"\n\t\toptional = true\n\t\tdefault = replace(coalesce(argument.role.value, + \"endpoints\"), \"endpoints\", \"service\")\n\t}\n\n\targument \"scrape_port_named_metrics\" + {\n\t\tcomment = \"Whether or not to automatically scrape endpoints that have + a port with 'metrics' in the name\"\n\t\toptional = true\n\t\tdefault = false\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: (.+))\"\n\t\toptional + = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment = \"A regex of metrics + to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" + {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + Targets From Docker Discovery\n\t*****************************************************************/\n\tdiscovery.kubernetes + \"annotation_metrics\" {\n\t\trole = coalesce(argument.role.value, \"endpoints\")\n\n\t\tselectors + {\n\t\t\trole = coalesce(argument.role.value, \"endpoints\")\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t*****************************************************************/\n\t// + filter metrics by kubernetes annotations\n\tdiscovery.relabel \"annotation_metrics_filter\" + {\n\t\ttargets = discovery.kubernetes.annotation_metrics.targets\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Targets to Keep or Drop\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare their metrics scraped or not\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/scrape: false\n\t\t//\n\t\t// the label prometheus.io/service-monitor: + \"false\" is a common label for headless services, when performing endpoint\n\t\t// + service discovery, if there is both a load-balanced service and headless service, + this can result in duplicate\n\t\t// scrapes if the name of the service is attached + as a label. any targets with this label or annotation set should be dropped\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"false\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, + \"endpoints\") + \"_label_prometheus_io_service_monitor\",\n\t\t\t]\n\t\t\tseparator + = \";\"\n\t\t\t// only allow empty or true, otherwise defaults to false\n\t\t\tregex + \ = \"^(?:;*)?(true)(;|true)*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\t// add a __tmp_scrape_port_named_metrics from + the argument.scrape_port_named_metrics\n\t\trule {\n\t\t\treplacement = format(\"%t\", + argument.scrape_port_named_metrics.value)\n\t\t\ttarget_label = \"__tmp_scrape_port_named_metrics\"\n\t\t}\n\n\t\t// + only keep targets that have scrape: true or \"metrics\" in the port name if the + argument scrape_port_named_metrics\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__tmp_scrape\",\n\t\t\t\t\"__tmp_scrape_port_named_metrics\",\n\t\t\t\t// + endpoints is the role and most meta labels started with \"endpoints\", however + the port name is an exception and starts with \"endpoint\"\n\t\t\t\t\"__meta_kubernetes_\" + + replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"endpoint\") + + \"_port_name\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(true;.*|(|true);true;(.*metrics.*))$\"\n\t\t}\n\n\t\t// + only keep targets where the pod is running or the pod_phase is empty and is not + an init container. This will only exist for role=\"pod\" or\n\t\t// potentially + role=\"endpoints\", if it is a service the value is empty and thus allowed to + pass, if it is an endpoint but not associated to a\n\t\t// pod but rather a static + IP or hostname, that could be outside of kubernetes allow endpoints to declare + what tenant their metrics should be\n\t\t// written to\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex + \ = \"^(?i)(Running|)$\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__meta_kubernetes_pod_ready\"]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\t\t// + if the container is an init container, drop it\n\t\trule {\n\t\t\taction = + \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_container_init\"]\n\t\t\tregex + \ = \"^(true)$\"\n\t\t}\n\n\t\t// allow resources to declare their metrics + the tenant their metrics should be sent to,\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/tenant: primary\n\t\t//\n\t\t// Note: This does not necessarily + have to be the actual tenantId, it can be a friendly name as well that is simply + used\n\t\t// to determine if the metrics should be gathered for the current + tenant\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_tenant\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_tenant\",\n\t\t\t]\n\t\t\tregex + = \"^(\" + coalesce(argument.tenant.value, \".*\") + \")$\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Scrape Metadata i.e. path, port, interval etc.\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare the protocol to use when collecting metrics, the default + value is \"http\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/scheme: + http\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"http\"\n\t\t\ttarget_label + = \"__scheme__\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scheme\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scheme\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?(https?).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scheme__\"\n\t\t}\n\n\t\t// allow resources + to declare the port to use when collecting metrics, the default value is the discovered + port from\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/port: 9090\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__address__\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_port\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_port\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^:]+)(?::\\\\d+)?;(\\\\d+)$\"\n\t\t\treplacement + \ = \"$1:$2\"\n\t\t\ttarget_label = \"__address__\"\n\t\t}\n\n\t\t// allow resources + to declare their the path to use when collecting their metrics, the default value + is \"/metrics\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/path: + /metrics/foo\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_path\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_path\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__metrics_path__\"\n\t\t}\n\n\t\t// allow resources + to declare how often their metrics should be collected, the default value is 1m,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/interval: 5m\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_interval\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_interval\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\t// allow + resources to declare the timeout of the scrape request, the default value is 10s,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/timeout: 30s\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_timeout\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_timeout\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Common Labels\n\t\t****************************************************************************************************************/\n\t\t// + set a source label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = \"kubernetes\"\n\t\t\ttarget_label = \"source\"\n\t\t}\n\n\t\t// set the cluster + label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = argument.cluster.value\n\t\t\ttarget_label + = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label + \ = \"namespace\"\n\t\t}\n\n\t\t// set the target name label i.e. service name, + pod name, etc.\n\t\t// if the role is endpoints, the first valued field is used + which would be __meta_kubernetes_pod_name, if the pod name is empty\n\t\t// then + the endpoint name would be used\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + argument.__pod_role.value + \"_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = + \"$1\"\n\t\t\ttarget_label = argument.__pod_role.value\n\t\t}\n\n\t\t// set a + default job label to be the namespace/pod_controller_name or namespace/service_name\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t\targument.__pod_role.value,\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^;]+)(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// if the controller + is a ReplicaSet, drop the hash from the end of the ReplicaSet\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_type\",\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:ReplicaSet);([^;]+);([^;]+)-.+$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// allow resources to + declare their the job label value to use when collecting their metrics, the default + value is \"\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/job: + integrations/kubernetes/cadvisor\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_job\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_job\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// set the app name if + specified as metadata labels \"app:\" or \"app.kubernetes.io/name:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"app\"\n\t\t}\n\n\t\t// + set the app component if specified as metadata labels \"component:\" or \"app.kubernetes.io/component:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_component\",\n\t\t\t]\n\t\t\tregex + \ = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"component\"\n\t\t}\n\n\t\t// set the version if specified as metadata labels + \"version:\" or \"app.kubernetes.io/version:\" or \"app_version:\"\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_version\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"version\"\n\t\t}\n\n\t\t// + set a workload label if the resource is a pod\n\t\t// example: grafana-agent-68nv9 + becomes DaemonSet/grafana-agent\n\t\trule {\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_kind\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"(.+);(.+)\"\n\t\t\treplacement = \"$1/$2\"\n\t\t\ttarget_label + = \"workload\"\n\t\t}\n\t\t// remove the hash from the ReplicaSet\n\t\trule {\n\t\t\tsource_labels + = [\"workload\"]\n\t\t\tregex = \"(ReplicaSet/.+)-.+\"\n\t\t\ttarget_label + \ = \"workload\"\n\t\t}\n\t}\n\n\t// only keep http targets\n\tdiscovery.relabel + \"http_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"http\"\n\t\t}\n\t}\n\n\t// only keep https targets\n\tdiscovery.relabel + \"https_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"https\"\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Scrape Labels Targets\n\t*****************************************************************/\n\t// + scrape http only targtetsa\n\tprometheus.scrape \"http_annotations\" {\n\t\ttargets + = discovery.relabel.http_annotations.output\n\n\t\tjob_name = \"annotation-metrics-http\"\n\t\tscheme + \ = \"http\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t// + scrape https only targtets\n\tprometheus.scrape \"https_annotations\" {\n\t\ttargets + = discovery.relabel.https_annotations.output\n\n\t\tjob_name = \"annotation-metrics-https\"\n\t\tscheme + \ = \"https\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\tbearer_token_file + = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n\n\t\ttls_config {\n\t\t\tca_file + \ = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n\t\t\tinsecure_skip_verify + = false\n\t\t\tserver_name = \"kubernetes\"\n\t\t}\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"annotations\" {\n\t\tforward_to = argument.forward_to.value\n\t\t// keep only + metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(.+)\")\n\t\t}\n\n\t\t// drop metrics that match the drop_metrics regex\n\t\trule + {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex + \ = coalesce(argument.drop_metrics.value, \"\")\n\t\t}\n\t}\n}\n" podmonitors-scrape.alloy: "/*\nModule Components: podmonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.podmonitors\n*/\n\ndeclare \"podmonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape PodMonitors\n\t*****************************************************************/\n\tprometheus.operator.podmonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.podmonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\t// selector {\n\t\t// \tmatch_expression {\n\t\t// \t\tkey = \"team\"\n\t\t// \t\toperator = \"In\"\n\t\t// \t\tvalues - \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n}\n" + \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"podmonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set the + cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" + rules-to-mimir.alloy: "/*\nModule Components: rules_to_mimir\nDescription: Auto + discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance.\n*/\n\ndeclare + \"rules_to_mimir\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + anonymous)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Mimir\n\t********************************************/\n\tmimir.rules.kubernetes + \"rules_to_mimir\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:8080\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\t// rule_selector {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= + \"true\",\n\t\t// \t}\n\t\t// }\n\t}\n}\n" servicemonitors-scrape.alloy: "/*\nModule Components: servicemonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.servicemonitors\n*/\n\ndeclare \"servicemonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape ServiceMonitors\n\t*****************************************************************/\n\tprometheus.operator.servicemonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.servicemonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering - {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n" + {\n\t\t\tenabled = true\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"servicemonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set + the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f namespace: monitoring-system --- apiVersion: v1 @@ -352,19 +694,21 @@ data: \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(ProfilessReceiver) where collected - logs should be forwarded to\"\n\t}\n\n\tdiscovery.kubernetes \"pyroscope_kubernetes\" - {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape config allows to define - annotations based scraping.\n\t//\n\t// For example the following annotations:\n\t//\n\t// - ```\n\t// profiles.grafana.com/memory.scrape: \"true\"\n\t// profiles.grafana.com/memory.port: - \"8080\"\n\t// profiles.grafana.com/cpu.scrape: \"true\"\n\t// profiles.grafana.com/cpu.port: - \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: \"true\"\n\t// profiles.grafana.com/goroutine.port: - \"8080\"\n\t// ```\n\t//\n\t// will scrape the `memory`, `cpu` and `goroutine` - profiles from the `8080` port of the pod.\n\t//\n\t// For more information see - https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel + logs should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\tdiscovery.kubernetes + \"pyroscope_kubernetes\" {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape + config allows to define annotations based scraping.\n\t//\n\t// For example the + following annotations:\n\t//\n\t// ```\n\t// profiles.grafana.com/memory.scrape: + \"true\"\n\t// profiles.grafana.com/memory.port: \"8080\"\n\t// profiles.grafana.com/cpu.scrape: + \"true\"\n\t// profiles.grafana.com/cpu.port: \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: + \"true\"\n\t// profiles.grafana.com/goroutine.port: \"8080\"\n\t// ```\n\t//\n\t// + will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port + of the pod.\n\t//\n\t// For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel \"kubernetes_pods\" {\n\t\ttargets = concat(discovery.kubernetes.pyroscope_kubernetes.targets)\n\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex \ = \"Pending|Succeeded|Failed|Completed\"\n\t\t}\n\n\t\trule {\n\t\t\taction - = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\trule + = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\trule @@ -583,7 +927,7 @@ data: {\n\t\t\t\tenabled = true\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 namespace: monitoring-system --- apiVersion: v1 @@ -593,40 +937,58 @@ data: Transformation\ndeclare \"process_and_transform\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"traces_forward_to\" {\n\t\tcomment = \"Must be a list(TracesReceiver) where - collected traces should be forwarded to\"\n\t}\n\n\targument \"logs_forward_to\" - {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be - forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment = \"Must - be a list(MetricsReceiver) where collected metrics should be forwarded to\"\n\t}\n\n\targument - \"cluster\" {\n\t\toptional = true\n\t\tdefault = \"k3d-k3s-codelab\"\n\t}\n\n\targument - \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4318\"\n\t}\n\n\targument - \"otlp_grpc_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* + collected traces should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\targument + \"logs_forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected + logs should be forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment + = \"Must be a list(MetricsReceiver) where collected metrics should be forwarded + to\"\n\t}\n\n\targument \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault + \ = \"0.0.0.0:4318\"\n\t}\n\n\targument \"otlp_grpc_endpoint\" {\n\t\toptional + = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* Jaeger for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.jaeger \"default\" {\n\t\tprotocols {\n\t\t\tgrpc {\n\t\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t\t}\n\n\t\t\tthrift_http {\n\t\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t\t}\n\n\t\t\tthrift_binary {\n\t\t\t\tendpoint = \"0.0.0.0:6832\"\n\t\t\t}\n\n\t\t\tthrift_compact {\n\t\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [otelcol.processor.resourcedetection.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + {\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* Otelcol for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.otlp - \"default\" {\n\t\tgrpc {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp - {\n\t\t\tendpoint = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics - = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection - \"default\" {\n\t\tdetectors = [\"env\"]\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces - = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes + \"default\" {\n\t\tdebug_metrics {\n\t\t\tdisable_high_cardinality_metrics = true\n\t\t}\n\n\t\tgrpc + {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp {\n\t\t\tendpoint + = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.resourcedetection.default.input]\n\t\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection + \"default\" {\n\t\tdetectors = [\"env\", \"system\"]\n\n\t\tsystem {\n\t\t\thostname_sources + = [\"os\"]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\n\t\t\tlogs + \ = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform + \"add_metric_datapoint_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements + {\n\t\t\tcontext = \"datapoint\"\n\t\t\tstatements = [\n\t\t\t\t\"set(attributes[\\\"deployment.environment\\\"], + resource.attributes[\\\"deployment.environment\\\"])\",\n\t\t\t\t\"set(attributes[\\\"service.version\\\"], + resource.attributes[\\\"service.version\\\"])\",\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes \"default\" {\n\t\textract {\n\t\t\tmetadata = [\n\t\t\t\t\"k8s.namespace.name\",\n\t\t\t\t\"k8s.pod.name\",\n\t\t\t\t\"k8s.deployment.name\",\n\t\t\t\t\"k8s.statefulset.name\",\n\t\t\t\t\"k8s.daemonset.name\",\n\t\t\t\t\"k8s.cronjob.name\",\n\t\t\t\t\"k8s.job.name\",\n\t\t\t\t\"k8s.node.name\",\n\t\t\t\t\"k8s.pod.uid\",\n\t\t\t\t\"k8s.pod.start_time\",\n\t\t\t]\n\t\t}\n\n\t\tpod_association {\n\t\t\tsource {\n\t\t\t\tfrom = \"connection\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t\ttraces - = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform - \"add_resource_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tlog_statements + {\n\t\t\tmetrics = [otelcol.processor.transform.default.input]\n\t\t\tlogs = + [otelcol.processor.transform.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.transform.default.input,\n\t\t\t\totelcol.connector.host_info.default.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.connector.host_info + \"default\" {\n\t\thost_identifiers = [\"k8s.node.name\"]\n\n\t\toutput {\n\t\t\tmetrics + = [otelcol.processor.batch.host_info_batch.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + \"host_info_batch\" {\n\t\toutput {\n\t\t\tmetrics = [otelcol.exporter.prometheus.host_info_metrics.input]\n\t\t}\n\t}\n\n\totelcol.exporter.prometheus + \"host_info_metrics\" {\n\t\tadd_metric_suffixes = false\n\t\tforward_to = + argument.metrics_forward_to.value\n\t}\n\n\totelcol.processor.transform \"default\" + {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements {\n\t\t\tcontext = + \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\tlog_statements {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])`,\n\t\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t\t`set(attributes[\"loki.resource.labels\"], \"pod, namespace, cluster, job\")`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements - {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], - \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter - \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.batch.default.input]\n\t\t\ttraces - = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements + {\n\t\t\tcontext = \"span\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces + \ = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter + \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\ttraces {\n\t\t\tspan = [\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/live\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] == \\\"/healthy\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/ready\\\"\",\n\t\t\t]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs + \ = [otelcol.processor.batch.default.input]\n\t\t\ttraces = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch \"default\" {\n\t\tsend_batch_size = 16384\n\t\tsend_batch_max_size = 0\n\t\ttimeout \ = \"5s\"\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\t\tlogs \ = [otelcol.processor.memory_limiter.default.input]\n\t\t\ttraces = [otelcol.processor.memory_limiter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.memory_limiter @@ -662,7 +1024,7 @@ data: \"alloy_traces_input\" {\n\t\tvalue = otelcol.processor.batch.default.input\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 namespace: monitoring-system --- apiVersion: v1 @@ -2263,19 +2625,19 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-55dd47md5f + name: alloy-config-dkdkkgdd64 name: config - configMap: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f name: modules-kubernetes-metrics - configMap: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k name: modules-kubernetes-logs - configMap: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 name: modules-kubernetes-traces - configMap: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 name: modules-kubernetes-profiles --- apiVersion: apps/v1 diff --git a/kubernetes/microservices-mode/metrics/metrics.alloy b/kubernetes/microservices-mode/metrics/metrics.alloy index 606728c9..b37ad416 100644 --- a/kubernetes/microservices-mode/metrics/metrics.alloy +++ b/kubernetes/microservices-mode/metrics/metrics.alloy @@ -24,16 +24,13 @@ import.file "metrics" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/metrics" } -metrics.podmonitors_scrape "kubernetes" { +metrics.annotations_scrape "kubernetes" { forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] + + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" } metrics.servicemonitors_scrape "kubernetes" { forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } - -// metrics.annotations_scrape "kubernetes" { -// label_prefix = "metrics.grafana.com" -// forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] -// scrape_interval = "15s" -// } diff --git a/kubernetes/microservices-mode/profiles/k8s-all-in-one.yaml b/kubernetes/microservices-mode/profiles/k8s-all-in-one.yaml index 4c70d325..0ef44e90 100644 --- a/kubernetes/microservices-mode/profiles/k8s-all-in-one.yaml +++ b/kubernetes/microservices-mode/profiles/k8s-all-in-one.yaml @@ -227,44 +227,43 @@ data: = coalesce(env(\"SELF_HOSTED_PROFILES_ENDPOINT_URL\"), \"http://nginx.gateway.svc:4040\")\n}\n\n/********************************************\n * Profiles\n ********************************************/\nimport.file \"profiles\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/profiles\"\n}\n\nprofiles.annotations_scrape \"kubernetes\" {\n\t// - annotation_prefix = \"profiles.grafana.com\"\n\tforward_to = [provider.self_hosted_stack.kubernetes.profiles_receiver]\n}\n\n/********************************************\n + + \"/kubernetes/profiles\"\n}\n\nprofiles.annotations_scrape \"kubernetes\" {\n\tcluster + = \"k3d-k3s-codelab\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.profiles_receiver]\n}\n\n/********************************************\n * Metrics\n ********************************************/\nimport.file \"metrics\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/metrics\"\n}\n\nmetrics.podmonitors_scrape \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape - \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" + + \"/kubernetes/metrics\"\n}\n\nmetrics.annotations_scrape \"kubernetes\" {\n\tcluster + \ = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-56888m5g8t + name: alloy-config-m5kk6k2f7k namespace: monitoring-system --- apiVersion: v1 data: annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: Scrapes targets for logs based on kubernetes Pod annotations\n\n Annotations:\n - \ logs.grafana.com/ingest: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare + \ logs.grafana.com/scrape: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this - can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\t// arguments - for kubernetes discovery\n\targument \"namespaces\" {\n\t\tcomment = \"The namespaces - to look for targets in (default: [\\\"kube-system\\\"] is all namespaces)\"\n\t\toptional - = true\n\t}\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\targument + \"cluster\" { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix to use (default: logs.grafana.com)\"\n\t\tdefault = \"logs.grafana.com\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional = true\n\t\tcomment \ = \"The logic is used to transform the annotation argument into a valid label name by removing unsupported characters.\"\n\t\tdefault = replace(replace(replace(coalesce(argument.annotation_prefix.value, \"logs.grafana.com\"), \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\t// - find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\n\t\tnamespaces - {\n\t\t\tnames = coalesce(argument.namespaces.value, [])\n\t\t}\n\t}\n\n\t// filter - logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" + find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\t}\n\n\t// + filter logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" {\n\t\ttargets = discovery.kubernetes.annotation_logs.targets\n\n\t\t// allow pods to declare their logs to be ingested or not, the default is true\n\t\t// - \ i.e. logs.grafana.com/ingest: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + \ i.e. logs.grafana.com/scrape: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\n\t\t// allow pods to declare what tenant their logs should be written to, the following annotation @@ -274,6 +273,8 @@ data: + argument.tenant.value + \")$\"\n\t\t}\n\n\t\t// set the instance label as the name of the worker node the pod is on\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_node_name\"]\n\t\t\ttarget_label = \"instance\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\t// set the pod label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\t// @@ -338,12 +339,11 @@ data: ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\toptional - = true\n\t}\n\n\targument \"cluster\" {\n\t\toptional = true\n\t}\n\n\tloki.source.kubernetes_events - \"cluster_events\" {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format + = true\n\t}\n\n\targument \"cluster\" { }\n\n\tloki.source.kubernetes_events \"cluster_events\" + {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format = \"logfmt\"\n\t\tforward_to = [loki.process.logs_service.receiver]\n\t}\n\n\tloki.process \"logs_service\" {\n\t\tstage.static_labels {\n\t\t\tvalues = {\n\t\t\t\tcluster - = coalesce(argument.cluster.value, \"k3d\"),\n\t\t\t}\n\t\t}\n\t\tforward_to = - argument.forward_to.value\n\t}\n}\n" + = argument.cluster.value,\n\t\t\t}\n\t\t}\n\t\tforward_to = argument.forward_to.value\n\t}\n}\n" keep-labels.alloy: "/*\nModule Components: keep_labels\nDescription: Pre-defined set of labels to keep, this stage should always be in-place as the previous relabeing\n \ stages make every pod label and annotation a label in the pipeline, @@ -365,46 +365,392 @@ data: = argument.keep_labels.value\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* EXPORTS\n\t*****************************************************************/\n\texport \"receiver\" {\n\t\tvalue = loki.process.keep_labels.receiver\n\t}\n}\n" + rules-to-loki.alloy: "/*\nModule Components: rules_to_loki\nDescription: Auto discovers + PrometheusRule Kubernetes resources and loads them into a Loki instance.\n*/\n\ndeclare + \"rules_to_loki\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + fake)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Loki\n\t********************************************/\n\tloki.rules.kubernetes + \"rules_to_loki\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:3100\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_loki= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\trule_selector {\n\t\t\tmatch_labels = {\n\t\t\t\tauto_rules_to_loki + = \"true\",\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k namespace: monitoring-system --- apiVersion: v1 data: + annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: + Scrapes targets for metrics based on kubernetes annotations\n\nNote: Every argument + except for \"forward_to\" is optional, and does have a defined default value. + \ However, the values for these\n arguments are not defined using the default + = \" ... \" argument syntax, but rather using the coalesce(argument.value, \" + ... \").\n This is because if the argument passed in from another consuming + module is set to null, the default = \" ... \" syntax will\n does not override + the value passed in, where coalesce() will return the first non-null value.\n\nKubernetes + Annotation Auto-Scraping\n------------------------------------------------------------------------------------------------------------------------------------\nThis + module is meant to be used to automatically scrape targets based on a certain + role and set of annotations. This module can be consumed\nmultiple times with + different roles. The supported roles are:\n\n - pod\n - service\n - endpoints\n\nTypically, + if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality + you would use role=\"endpoints\", if\nmimicking the behavior of the PodMonitor + functionality you would use role=\"pod\". It is important to understand that + with endpoints,\nthe target is typically going to be a pod, and whatever annotations + that are set on the service will automatically be propagated to the\nendpoints. + \ This is why the role \"endpoints\" is used, because it will scrape the pod, + but also consider the service annotations. Using\nrole=\"endpoints\", which scrape + each endpoint associated to the service. If role=\"service\" is used, it will + only scrape the service, only\nhitting one of the endpoints associated to the + service.\n\nThis is where you must consider your scraping strategy, for example + if you scrape a service like \"kube-state-metrics\" using\nrole=\"endpoints\" + you should only have a single replica of the kube-state-metrics pod, if you have + multiple replicas, you should use\nrole=\"service\" or a separate non-annotation + job completely. Scraping a service instead of endpoints, is typically a rare + use case, but\nit is supported.\n\nThere are other considerations for using annotation + based scraping as well, which is metric relabeling rules that happen post scrape. + \ If\nyou have a target that you want to apply a bunch of relabelings to or a + very large metrics response payload, performance wise it will be\nbetter to have + a separate job for that target, rather than using use annotations. As every targert + will go through the ssame relabeling.\nTypical deployment strategies/options would + be:\n\nOption #1 (recommended):\n - Annotation Scraping for role=\"endpoints\"\n + \ - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, + etc.) or large metric payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, + kube-apiserver, kube-scheduler, etc.)\n\nOption #2:\n - Annotation Scraping for + role=\"pod\"\n - Annotation Scraping for role=\"service\" (i.e. kube-state-metrics, + node-exporter, etc.)\n - Separate Jobs for specific use cases or large metric + payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, + kube-scheduler, etc.)\n\nAt no point should you use role=\"endpoints\" and role=\"pod\" + together, as this will result in duplicate targets being scraped, thus\ngenerating + duplicate metrics. If you want to scrape both the pod and the service, use Option + #2.\n\nEach port attached to an service/pod/endpoint is an eligible target, oftentimes + it will have multiple ports.\nThere may be instances when you want to scrape all + ports or some ports and not others. To support this\nthe following annotations + are available:\n\n metrics.grafana.com/scrape: true\n\nthe default scraping scheme + is http, this can be specified as a single value which would override, the schema + being used for all\nports attached to the target:\n\n metrics.grafana.com/scheme: + https\n\nthe default path to scrape is /metrics, this can be specified as a single + value which would override, the scrape path being used\nfor all ports attached + to the target:\n\n metrics.grafana.com/path: /metrics/some_path\n\nthe default + port to scrape is the target port, this can be specified as a single value which + would override the scrape port being\nused for all ports attached to the target, + note that even if aan target had multiple targets, the relabel_config targets + are\ndeduped before scraping:\n\n metrics.grafana.com/port: 8080\n\nthe default + interval to scrape is 1m, this can be specified as a single value which would + override, the scrape interval being used\nfor all ports attached to the target:\n\n + \ metrics.grafana.com/interval: 5m\n\nthe default timeout for scraping is 10s, + this can be specified as a single value which would override, the scrape interval + being\nused for all ports attached to the target:\n\n metrics.grafana.com/timeout: + 30s\n\nthe default job is namespace/{{ service name }} or namespace/{{ controller_name + }} depending on the role, there may be instances\nin which a different job name + is required because of a set of dashboards, rules, etc. to support this there + is a job annotation\nwhich will override the default value:\n\n metrics.grafana.com/job: + integrations/kubernetes/kube-state-metrics\n*/\n\ndeclare \"annotations_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment + \ = \"The tenant to filter metrics to. This does not have to be the tenantId, + this is the value to look for in the metrics.agent.grafana.com/tenant annotation, + and this can be a regex.\"\n\t\toptional = true\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + to use (default: metrics.grafana.com)\"\n\t\tdefault = \"metrics.grafana.com\"\n\t\toptional + = true\n\t}\n\n\targument \"role\" {\n\t\tcomment = \"The role to use when looking + for targets to scrape via annotations, can be: endpoints, service, pod (default: + endpoints)\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional + = true\n\t\tcomment = \"The logic is used to transform the annotation argument + into a valid label name by removing unsupported characters.\"\n\t\tdefault = + replace(replace(replace(coalesce(argument.annotation_prefix.value, \"metrics.grafana.com\"), + \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\targument \"__pod_role\" + {\n\t\tcomment = \"Most annotation targets service or pod that is all you want, + however if the role is endpoints you want the pod\"\n\t\toptional = true\n\t\tdefault + \ = replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"pod\")\n\t}\n\n\targument + \"__service_role\" {\n\t\tcomment = \"Most annotation targets service or pod + that is all you want, however if the role is endpoints you we also want to consider + service annotations\"\n\t\toptional = true\n\t\tdefault = replace(coalesce(argument.role.value, + \"endpoints\"), \"endpoints\", \"service\")\n\t}\n\n\targument \"scrape_port_named_metrics\" + {\n\t\tcomment = \"Whether or not to automatically scrape endpoints that have + a port with 'metrics' in the name\"\n\t\toptional = true\n\t\tdefault = false\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: (.+))\"\n\t\toptional + = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment = \"A regex of metrics + to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" + {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + Targets From Docker Discovery\n\t*****************************************************************/\n\tdiscovery.kubernetes + \"annotation_metrics\" {\n\t\trole = coalesce(argument.role.value, \"endpoints\")\n\n\t\tselectors + {\n\t\t\trole = coalesce(argument.role.value, \"endpoints\")\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t*****************************************************************/\n\t// + filter metrics by kubernetes annotations\n\tdiscovery.relabel \"annotation_metrics_filter\" + {\n\t\ttargets = discovery.kubernetes.annotation_metrics.targets\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Targets to Keep or Drop\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare their metrics scraped or not\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/scrape: false\n\t\t//\n\t\t// the label prometheus.io/service-monitor: + \"false\" is a common label for headless services, when performing endpoint\n\t\t// + service discovery, if there is both a load-balanced service and headless service, + this can result in duplicate\n\t\t// scrapes if the name of the service is attached + as a label. any targets with this label or annotation set should be dropped\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"false\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, + \"endpoints\") + \"_label_prometheus_io_service_monitor\",\n\t\t\t]\n\t\t\tseparator + = \";\"\n\t\t\t// only allow empty or true, otherwise defaults to false\n\t\t\tregex + \ = \"^(?:;*)?(true)(;|true)*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\t// add a __tmp_scrape_port_named_metrics from + the argument.scrape_port_named_metrics\n\t\trule {\n\t\t\treplacement = format(\"%t\", + argument.scrape_port_named_metrics.value)\n\t\t\ttarget_label = \"__tmp_scrape_port_named_metrics\"\n\t\t}\n\n\t\t// + only keep targets that have scrape: true or \"metrics\" in the port name if the + argument scrape_port_named_metrics\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__tmp_scrape\",\n\t\t\t\t\"__tmp_scrape_port_named_metrics\",\n\t\t\t\t// + endpoints is the role and most meta labels started with \"endpoints\", however + the port name is an exception and starts with \"endpoint\"\n\t\t\t\t\"__meta_kubernetes_\" + + replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"endpoint\") + + \"_port_name\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(true;.*|(|true);true;(.*metrics.*))$\"\n\t\t}\n\n\t\t// + only keep targets where the pod is running or the pod_phase is empty and is not + an init container. This will only exist for role=\"pod\" or\n\t\t// potentially + role=\"endpoints\", if it is a service the value is empty and thus allowed to + pass, if it is an endpoint but not associated to a\n\t\t// pod but rather a static + IP or hostname, that could be outside of kubernetes allow endpoints to declare + what tenant their metrics should be\n\t\t// written to\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex + \ = \"^(?i)(Running|)$\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__meta_kubernetes_pod_ready\"]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\t\t// + if the container is an init container, drop it\n\t\trule {\n\t\t\taction = + \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_container_init\"]\n\t\t\tregex + \ = \"^(true)$\"\n\t\t}\n\n\t\t// allow resources to declare their metrics + the tenant their metrics should be sent to,\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/tenant: primary\n\t\t//\n\t\t// Note: This does not necessarily + have to be the actual tenantId, it can be a friendly name as well that is simply + used\n\t\t// to determine if the metrics should be gathered for the current + tenant\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_tenant\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_tenant\",\n\t\t\t]\n\t\t\tregex + = \"^(\" + coalesce(argument.tenant.value, \".*\") + \")$\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Scrape Metadata i.e. path, port, interval etc.\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare the protocol to use when collecting metrics, the default + value is \"http\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/scheme: + http\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"http\"\n\t\t\ttarget_label + = \"__scheme__\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scheme\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scheme\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?(https?).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scheme__\"\n\t\t}\n\n\t\t// allow resources + to declare the port to use when collecting metrics, the default value is the discovered + port from\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/port: 9090\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__address__\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_port\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_port\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^:]+)(?::\\\\d+)?;(\\\\d+)$\"\n\t\t\treplacement + \ = \"$1:$2\"\n\t\t\ttarget_label = \"__address__\"\n\t\t}\n\n\t\t// allow resources + to declare their the path to use when collecting their metrics, the default value + is \"/metrics\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/path: + /metrics/foo\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_path\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_path\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__metrics_path__\"\n\t\t}\n\n\t\t// allow resources + to declare how often their metrics should be collected, the default value is 1m,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/interval: 5m\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_interval\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_interval\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\t// allow + resources to declare the timeout of the scrape request, the default value is 10s,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/timeout: 30s\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_timeout\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_timeout\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Common Labels\n\t\t****************************************************************************************************************/\n\t\t// + set a source label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = \"kubernetes\"\n\t\t\ttarget_label = \"source\"\n\t\t}\n\n\t\t// set the cluster + label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = argument.cluster.value\n\t\t\ttarget_label + = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label + \ = \"namespace\"\n\t\t}\n\n\t\t// set the target name label i.e. service name, + pod name, etc.\n\t\t// if the role is endpoints, the first valued field is used + which would be __meta_kubernetes_pod_name, if the pod name is empty\n\t\t// then + the endpoint name would be used\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + argument.__pod_role.value + \"_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = + \"$1\"\n\t\t\ttarget_label = argument.__pod_role.value\n\t\t}\n\n\t\t// set a + default job label to be the namespace/pod_controller_name or namespace/service_name\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t\targument.__pod_role.value,\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^;]+)(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// if the controller + is a ReplicaSet, drop the hash from the end of the ReplicaSet\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_type\",\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:ReplicaSet);([^;]+);([^;]+)-.+$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// allow resources to + declare their the job label value to use when collecting their metrics, the default + value is \"\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/job: + integrations/kubernetes/cadvisor\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_job\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_job\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// set the app name if + specified as metadata labels \"app:\" or \"app.kubernetes.io/name:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"app\"\n\t\t}\n\n\t\t// + set the app component if specified as metadata labels \"component:\" or \"app.kubernetes.io/component:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_component\",\n\t\t\t]\n\t\t\tregex + \ = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"component\"\n\t\t}\n\n\t\t// set the version if specified as metadata labels + \"version:\" or \"app.kubernetes.io/version:\" or \"app_version:\"\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_version\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"version\"\n\t\t}\n\n\t\t// + set a workload label if the resource is a pod\n\t\t// example: grafana-agent-68nv9 + becomes DaemonSet/grafana-agent\n\t\trule {\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_kind\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"(.+);(.+)\"\n\t\t\treplacement = \"$1/$2\"\n\t\t\ttarget_label + = \"workload\"\n\t\t}\n\t\t// remove the hash from the ReplicaSet\n\t\trule {\n\t\t\tsource_labels + = [\"workload\"]\n\t\t\tregex = \"(ReplicaSet/.+)-.+\"\n\t\t\ttarget_label + \ = \"workload\"\n\t\t}\n\t}\n\n\t// only keep http targets\n\tdiscovery.relabel + \"http_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"http\"\n\t\t}\n\t}\n\n\t// only keep https targets\n\tdiscovery.relabel + \"https_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"https\"\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Scrape Labels Targets\n\t*****************************************************************/\n\t// + scrape http only targtetsa\n\tprometheus.scrape \"http_annotations\" {\n\t\ttargets + = discovery.relabel.http_annotations.output\n\n\t\tjob_name = \"annotation-metrics-http\"\n\t\tscheme + \ = \"http\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t// + scrape https only targtets\n\tprometheus.scrape \"https_annotations\" {\n\t\ttargets + = discovery.relabel.https_annotations.output\n\n\t\tjob_name = \"annotation-metrics-https\"\n\t\tscheme + \ = \"https\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\tbearer_token_file + = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n\n\t\ttls_config {\n\t\t\tca_file + \ = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n\t\t\tinsecure_skip_verify + = false\n\t\t\tserver_name = \"kubernetes\"\n\t\t}\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"annotations\" {\n\t\tforward_to = argument.forward_to.value\n\t\t// keep only + metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(.+)\")\n\t\t}\n\n\t\t// drop metrics that match the drop_metrics regex\n\t\trule + {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex + \ = coalesce(argument.drop_metrics.value, \"\")\n\t\t}\n\t}\n}\n" podmonitors-scrape.alloy: "/*\nModule Components: podmonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.podmonitors\n*/\n\ndeclare \"podmonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape PodMonitors\n\t*****************************************************************/\n\tprometheus.operator.podmonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.podmonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\t// selector {\n\t\t// \tmatch_expression {\n\t\t// \t\tkey = \"team\"\n\t\t// \t\toperator = \"In\"\n\t\t// \t\tvalues - \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n}\n" + \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"podmonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set the + cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" + rules-to-mimir.alloy: "/*\nModule Components: rules_to_mimir\nDescription: Auto + discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance.\n*/\n\ndeclare + \"rules_to_mimir\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + anonymous)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Mimir\n\t********************************************/\n\tmimir.rules.kubernetes + \"rules_to_mimir\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:8080\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\t// rule_selector {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= + \"true\",\n\t\t// \t}\n\t\t// }\n\t}\n}\n" servicemonitors-scrape.alloy: "/*\nModule Components: servicemonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.servicemonitors\n*/\n\ndeclare \"servicemonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape ServiceMonitors\n\t*****************************************************************/\n\tprometheus.operator.servicemonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.servicemonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering - {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n" + {\n\t\t\tenabled = true\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"servicemonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set + the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f namespace: monitoring-system --- apiVersion: v1 @@ -414,19 +760,21 @@ data: \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(ProfilessReceiver) where collected - logs should be forwarded to\"\n\t}\n\n\tdiscovery.kubernetes \"pyroscope_kubernetes\" - {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape config allows to define - annotations based scraping.\n\t//\n\t// For example the following annotations:\n\t//\n\t// - ```\n\t// profiles.grafana.com/memory.scrape: \"true\"\n\t// profiles.grafana.com/memory.port: - \"8080\"\n\t// profiles.grafana.com/cpu.scrape: \"true\"\n\t// profiles.grafana.com/cpu.port: - \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: \"true\"\n\t// profiles.grafana.com/goroutine.port: - \"8080\"\n\t// ```\n\t//\n\t// will scrape the `memory`, `cpu` and `goroutine` - profiles from the `8080` port of the pod.\n\t//\n\t// For more information see - https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel + logs should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\tdiscovery.kubernetes + \"pyroscope_kubernetes\" {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape + config allows to define annotations based scraping.\n\t//\n\t// For example the + following annotations:\n\t//\n\t// ```\n\t// profiles.grafana.com/memory.scrape: + \"true\"\n\t// profiles.grafana.com/memory.port: \"8080\"\n\t// profiles.grafana.com/cpu.scrape: + \"true\"\n\t// profiles.grafana.com/cpu.port: \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: + \"true\"\n\t// profiles.grafana.com/goroutine.port: \"8080\"\n\t// ```\n\t//\n\t// + will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port + of the pod.\n\t//\n\t// For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel \"kubernetes_pods\" {\n\t\ttargets = concat(discovery.kubernetes.pyroscope_kubernetes.targets)\n\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex \ = \"Pending|Succeeded|Failed|Completed\"\n\t\t}\n\n\t\trule {\n\t\t\taction - = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\trule + = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\trule @@ -645,7 +993,7 @@ data: {\n\t\t\t\tenabled = true\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 namespace: monitoring-system --- apiVersion: v1 @@ -655,40 +1003,58 @@ data: Transformation\ndeclare \"process_and_transform\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"traces_forward_to\" {\n\t\tcomment = \"Must be a list(TracesReceiver) where - collected traces should be forwarded to\"\n\t}\n\n\targument \"logs_forward_to\" - {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be - forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment = \"Must - be a list(MetricsReceiver) where collected metrics should be forwarded to\"\n\t}\n\n\targument - \"cluster\" {\n\t\toptional = true\n\t\tdefault = \"k3d-k3s-codelab\"\n\t}\n\n\targument - \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4318\"\n\t}\n\n\targument - \"otlp_grpc_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* + collected traces should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\targument + \"logs_forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected + logs should be forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment + = \"Must be a list(MetricsReceiver) where collected metrics should be forwarded + to\"\n\t}\n\n\targument \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault + \ = \"0.0.0.0:4318\"\n\t}\n\n\targument \"otlp_grpc_endpoint\" {\n\t\toptional + = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* Jaeger for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.jaeger \"default\" {\n\t\tprotocols {\n\t\t\tgrpc {\n\t\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t\t}\n\n\t\t\tthrift_http {\n\t\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t\t}\n\n\t\t\tthrift_binary {\n\t\t\t\tendpoint = \"0.0.0.0:6832\"\n\t\t\t}\n\n\t\t\tthrift_compact {\n\t\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [otelcol.processor.resourcedetection.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + {\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* Otelcol for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.otlp - \"default\" {\n\t\tgrpc {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp - {\n\t\t\tendpoint = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics - = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection - \"default\" {\n\t\tdetectors = [\"env\"]\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces - = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes + \"default\" {\n\t\tdebug_metrics {\n\t\t\tdisable_high_cardinality_metrics = true\n\t\t}\n\n\t\tgrpc + {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp {\n\t\t\tendpoint + = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.resourcedetection.default.input]\n\t\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection + \"default\" {\n\t\tdetectors = [\"env\", \"system\"]\n\n\t\tsystem {\n\t\t\thostname_sources + = [\"os\"]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\n\t\t\tlogs + \ = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform + \"add_metric_datapoint_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements + {\n\t\t\tcontext = \"datapoint\"\n\t\t\tstatements = [\n\t\t\t\t\"set(attributes[\\\"deployment.environment\\\"], + resource.attributes[\\\"deployment.environment\\\"])\",\n\t\t\t\t\"set(attributes[\\\"service.version\\\"], + resource.attributes[\\\"service.version\\\"])\",\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes \"default\" {\n\t\textract {\n\t\t\tmetadata = [\n\t\t\t\t\"k8s.namespace.name\",\n\t\t\t\t\"k8s.pod.name\",\n\t\t\t\t\"k8s.deployment.name\",\n\t\t\t\t\"k8s.statefulset.name\",\n\t\t\t\t\"k8s.daemonset.name\",\n\t\t\t\t\"k8s.cronjob.name\",\n\t\t\t\t\"k8s.job.name\",\n\t\t\t\t\"k8s.node.name\",\n\t\t\t\t\"k8s.pod.uid\",\n\t\t\t\t\"k8s.pod.start_time\",\n\t\t\t]\n\t\t}\n\n\t\tpod_association {\n\t\t\tsource {\n\t\t\t\tfrom = \"connection\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t\ttraces - = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform - \"add_resource_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tlog_statements + {\n\t\t\tmetrics = [otelcol.processor.transform.default.input]\n\t\t\tlogs = + [otelcol.processor.transform.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.transform.default.input,\n\t\t\t\totelcol.connector.host_info.default.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.connector.host_info + \"default\" {\n\t\thost_identifiers = [\"k8s.node.name\"]\n\n\t\toutput {\n\t\t\tmetrics + = [otelcol.processor.batch.host_info_batch.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + \"host_info_batch\" {\n\t\toutput {\n\t\t\tmetrics = [otelcol.exporter.prometheus.host_info_metrics.input]\n\t\t}\n\t}\n\n\totelcol.exporter.prometheus + \"host_info_metrics\" {\n\t\tadd_metric_suffixes = false\n\t\tforward_to = + argument.metrics_forward_to.value\n\t}\n\n\totelcol.processor.transform \"default\" + {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements {\n\t\t\tcontext = + \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\tlog_statements {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])`,\n\t\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t\t`set(attributes[\"loki.resource.labels\"], \"pod, namespace, cluster, job\")`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements - {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], - \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter - \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.batch.default.input]\n\t\t\ttraces - = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements + {\n\t\t\tcontext = \"span\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces + \ = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter + \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\ttraces {\n\t\t\tspan = [\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/live\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] == \\\"/healthy\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/ready\\\"\",\n\t\t\t]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs + \ = [otelcol.processor.batch.default.input]\n\t\t\ttraces = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch \"default\" {\n\t\tsend_batch_size = 16384\n\t\tsend_batch_max_size = 0\n\t\ttimeout \ = \"5s\"\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\t\tlogs \ = [otelcol.processor.memory_limiter.default.input]\n\t\t\ttraces = [otelcol.processor.memory_limiter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.memory_limiter @@ -724,7 +1090,7 @@ data: \"alloy_traces_input\" {\n\t\tvalue = otelcol.processor.batch.default.input\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 namespace: monitoring-system --- apiVersion: v1 @@ -1997,19 +2363,19 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-56888m5g8t + name: alloy-config-m5kk6k2f7k name: config - configMap: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f name: modules-kubernetes-metrics - configMap: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k name: modules-kubernetes-logs - configMap: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 name: modules-kubernetes-traces - configMap: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 name: modules-kubernetes-profiles --- apiVersion: apps/v1 diff --git a/kubernetes/microservices-mode/profiles/profiles.alloy b/kubernetes/microservices-mode/profiles/profiles.alloy index cf0354fa..8828dd5b 100644 --- a/kubernetes/microservices-mode/profiles/profiles.alloy +++ b/kubernetes/microservices-mode/profiles/profiles.alloy @@ -26,7 +26,8 @@ import.file "profiles" { } profiles.annotations_scrape "kubernetes" { - // annotation_prefix = "profiles.grafana.com" + cluster = "k3d-k3s-codelab" + forward_to = [provider.self_hosted_stack.kubernetes.profiles_receiver] } @@ -37,10 +38,16 @@ import.file "metrics" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/metrics" } -metrics.podmonitors_scrape "kubernetes" { +metrics.annotations_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } metrics.servicemonitors_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } diff --git a/kubernetes/microservices-mode/traces/k8s-all-in-one.yaml b/kubernetes/microservices-mode/traces/k8s-all-in-one.yaml index 8a4d4ca8..87afbc54 100644 --- a/kubernetes/microservices-mode/traces/k8s-all-in-one.yaml +++ b/kubernetes/microservices-mode/traces/k8s-all-in-one.yaml @@ -236,47 +236,47 @@ data: * Traces\n ********************************************/\nimport.file \"traces\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/traces\"\n}\n\n// traces Processing And Transformation process_and_transform\ntraces.process_and_transform - \"kubernetes\" {\n\tmetrics_forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\tlogs_forward_to + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\n\tmetrics_forward_to + = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\tlogs_forward_to \ = [provider.self_hosted_stack.kubernetes.logs_receiver]\n\ttraces_forward_to \ = [provider.self_hosted_stack.kubernetes.traces_receiver]\n}\n\ntracing {\n\t// Write all spans. Don't do this in production!\n\tsampling_fraction = 1\n\n\t// Forward Alloy internal spans to traces process.\n\twrite_to = [traces.process_and_transform.kubernetes.alloy_traces_input]\n}\n\n/********************************************\n * Metrics\n ********************************************/\nimport.file \"metrics\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/metrics\"\n}\n\nmetrics.podmonitors_scrape \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape - \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" + + \"/kubernetes/metrics\"\n}\n\nmetrics.annotations_scrape \"kubernetes\" {\n\tcluster + \ = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-t69dk2g68h + name: alloy-config-284mhm8bfc namespace: monitoring-system --- apiVersion: v1 data: annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: Scrapes targets for logs based on kubernetes Pod annotations\n\n Annotations:\n - \ logs.grafana.com/ingest: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare + \ logs.grafana.com/scrape: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this - can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\t// arguments - for kubernetes discovery\n\targument \"namespaces\" {\n\t\tcomment = \"The namespaces - to look for targets in (default: [\\\"kube-system\\\"] is all namespaces)\"\n\t\toptional - = true\n\t}\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\targument + \"cluster\" { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix to use (default: logs.grafana.com)\"\n\t\tdefault = \"logs.grafana.com\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional = true\n\t\tcomment \ = \"The logic is used to transform the annotation argument into a valid label name by removing unsupported characters.\"\n\t\tdefault = replace(replace(replace(coalesce(argument.annotation_prefix.value, \"logs.grafana.com\"), \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\t// - find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\n\t\tnamespaces - {\n\t\t\tnames = coalesce(argument.namespaces.value, [])\n\t\t}\n\t}\n\n\t// filter - logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" + find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\t}\n\n\t// + filter logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" {\n\t\ttargets = discovery.kubernetes.annotation_logs.targets\n\n\t\t// allow pods to declare their logs to be ingested or not, the default is true\n\t\t// - \ i.e. logs.grafana.com/ingest: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + \ i.e. logs.grafana.com/scrape: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\n\t\t// allow pods to declare what tenant their logs should be written to, the following annotation @@ -286,6 +286,8 @@ data: + argument.tenant.value + \")$\"\n\t\t}\n\n\t\t// set the instance label as the name of the worker node the pod is on\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_node_name\"]\n\t\t\ttarget_label = \"instance\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\t// set the pod label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\t// @@ -350,12 +352,11 @@ data: ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\toptional - = true\n\t}\n\n\targument \"cluster\" {\n\t\toptional = true\n\t}\n\n\tloki.source.kubernetes_events - \"cluster_events\" {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format + = true\n\t}\n\n\targument \"cluster\" { }\n\n\tloki.source.kubernetes_events \"cluster_events\" + {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format = \"logfmt\"\n\t\tforward_to = [loki.process.logs_service.receiver]\n\t}\n\n\tloki.process \"logs_service\" {\n\t\tstage.static_labels {\n\t\t\tvalues = {\n\t\t\t\tcluster - = coalesce(argument.cluster.value, \"k3d\"),\n\t\t\t}\n\t\t}\n\t\tforward_to = - argument.forward_to.value\n\t}\n}\n" + = argument.cluster.value,\n\t\t\t}\n\t\t}\n\t\tforward_to = argument.forward_to.value\n\t}\n}\n" keep-labels.alloy: "/*\nModule Components: keep_labels\nDescription: Pre-defined set of labels to keep, this stage should always be in-place as the previous relabeing\n \ stages make every pod label and annotation a label in the pipeline, @@ -377,46 +378,392 @@ data: = argument.keep_labels.value\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* EXPORTS\n\t*****************************************************************/\n\texport \"receiver\" {\n\t\tvalue = loki.process.keep_labels.receiver\n\t}\n}\n" + rules-to-loki.alloy: "/*\nModule Components: rules_to_loki\nDescription: Auto discovers + PrometheusRule Kubernetes resources and loads them into a Loki instance.\n*/\n\ndeclare + \"rules_to_loki\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + fake)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Loki\n\t********************************************/\n\tloki.rules.kubernetes + \"rules_to_loki\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:3100\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_loki= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\trule_selector {\n\t\t\tmatch_labels = {\n\t\t\t\tauto_rules_to_loki + = \"true\",\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k namespace: monitoring-system --- apiVersion: v1 data: + annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: + Scrapes targets for metrics based on kubernetes annotations\n\nNote: Every argument + except for \"forward_to\" is optional, and does have a defined default value. + \ However, the values for these\n arguments are not defined using the default + = \" ... \" argument syntax, but rather using the coalesce(argument.value, \" + ... \").\n This is because if the argument passed in from another consuming + module is set to null, the default = \" ... \" syntax will\n does not override + the value passed in, where coalesce() will return the first non-null value.\n\nKubernetes + Annotation Auto-Scraping\n------------------------------------------------------------------------------------------------------------------------------------\nThis + module is meant to be used to automatically scrape targets based on a certain + role and set of annotations. This module can be consumed\nmultiple times with + different roles. The supported roles are:\n\n - pod\n - service\n - endpoints\n\nTypically, + if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality + you would use role=\"endpoints\", if\nmimicking the behavior of the PodMonitor + functionality you would use role=\"pod\". It is important to understand that + with endpoints,\nthe target is typically going to be a pod, and whatever annotations + that are set on the service will automatically be propagated to the\nendpoints. + \ This is why the role \"endpoints\" is used, because it will scrape the pod, + but also consider the service annotations. Using\nrole=\"endpoints\", which scrape + each endpoint associated to the service. If role=\"service\" is used, it will + only scrape the service, only\nhitting one of the endpoints associated to the + service.\n\nThis is where you must consider your scraping strategy, for example + if you scrape a service like \"kube-state-metrics\" using\nrole=\"endpoints\" + you should only have a single replica of the kube-state-metrics pod, if you have + multiple replicas, you should use\nrole=\"service\" or a separate non-annotation + job completely. Scraping a service instead of endpoints, is typically a rare + use case, but\nit is supported.\n\nThere are other considerations for using annotation + based scraping as well, which is metric relabeling rules that happen post scrape. + \ If\nyou have a target that you want to apply a bunch of relabelings to or a + very large metrics response payload, performance wise it will be\nbetter to have + a separate job for that target, rather than using use annotations. As every targert + will go through the ssame relabeling.\nTypical deployment strategies/options would + be:\n\nOption #1 (recommended):\n - Annotation Scraping for role=\"endpoints\"\n + \ - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, + etc.) or large metric payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, + kube-apiserver, kube-scheduler, etc.)\n\nOption #2:\n - Annotation Scraping for + role=\"pod\"\n - Annotation Scraping for role=\"service\" (i.e. kube-state-metrics, + node-exporter, etc.)\n - Separate Jobs for specific use cases or large metric + payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, + kube-scheduler, etc.)\n\nAt no point should you use role=\"endpoints\" and role=\"pod\" + together, as this will result in duplicate targets being scraped, thus\ngenerating + duplicate metrics. If you want to scrape both the pod and the service, use Option + #2.\n\nEach port attached to an service/pod/endpoint is an eligible target, oftentimes + it will have multiple ports.\nThere may be instances when you want to scrape all + ports or some ports and not others. To support this\nthe following annotations + are available:\n\n metrics.grafana.com/scrape: true\n\nthe default scraping scheme + is http, this can be specified as a single value which would override, the schema + being used for all\nports attached to the target:\n\n metrics.grafana.com/scheme: + https\n\nthe default path to scrape is /metrics, this can be specified as a single + value which would override, the scrape path being used\nfor all ports attached + to the target:\n\n metrics.grafana.com/path: /metrics/some_path\n\nthe default + port to scrape is the target port, this can be specified as a single value which + would override the scrape port being\nused for all ports attached to the target, + note that even if aan target had multiple targets, the relabel_config targets + are\ndeduped before scraping:\n\n metrics.grafana.com/port: 8080\n\nthe default + interval to scrape is 1m, this can be specified as a single value which would + override, the scrape interval being used\nfor all ports attached to the target:\n\n + \ metrics.grafana.com/interval: 5m\n\nthe default timeout for scraping is 10s, + this can be specified as a single value which would override, the scrape interval + being\nused for all ports attached to the target:\n\n metrics.grafana.com/timeout: + 30s\n\nthe default job is namespace/{{ service name }} or namespace/{{ controller_name + }} depending on the role, there may be instances\nin which a different job name + is required because of a set of dashboards, rules, etc. to support this there + is a job annotation\nwhich will override the default value:\n\n metrics.grafana.com/job: + integrations/kubernetes/kube-state-metrics\n*/\n\ndeclare \"annotations_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment + \ = \"The tenant to filter metrics to. This does not have to be the tenantId, + this is the value to look for in the metrics.agent.grafana.com/tenant annotation, + and this can be a regex.\"\n\t\toptional = true\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + to use (default: metrics.grafana.com)\"\n\t\tdefault = \"metrics.grafana.com\"\n\t\toptional + = true\n\t}\n\n\targument \"role\" {\n\t\tcomment = \"The role to use when looking + for targets to scrape via annotations, can be: endpoints, service, pod (default: + endpoints)\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional + = true\n\t\tcomment = \"The logic is used to transform the annotation argument + into a valid label name by removing unsupported characters.\"\n\t\tdefault = + replace(replace(replace(coalesce(argument.annotation_prefix.value, \"metrics.grafana.com\"), + \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\targument \"__pod_role\" + {\n\t\tcomment = \"Most annotation targets service or pod that is all you want, + however if the role is endpoints you want the pod\"\n\t\toptional = true\n\t\tdefault + \ = replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"pod\")\n\t}\n\n\targument + \"__service_role\" {\n\t\tcomment = \"Most annotation targets service or pod + that is all you want, however if the role is endpoints you we also want to consider + service annotations\"\n\t\toptional = true\n\t\tdefault = replace(coalesce(argument.role.value, + \"endpoints\"), \"endpoints\", \"service\")\n\t}\n\n\targument \"scrape_port_named_metrics\" + {\n\t\tcomment = \"Whether or not to automatically scrape endpoints that have + a port with 'metrics' in the name\"\n\t\toptional = true\n\t\tdefault = false\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: (.+))\"\n\t\toptional + = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment = \"A regex of metrics + to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" + {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + Targets From Docker Discovery\n\t*****************************************************************/\n\tdiscovery.kubernetes + \"annotation_metrics\" {\n\t\trole = coalesce(argument.role.value, \"endpoints\")\n\n\t\tselectors + {\n\t\t\trole = coalesce(argument.role.value, \"endpoints\")\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t*****************************************************************/\n\t// + filter metrics by kubernetes annotations\n\tdiscovery.relabel \"annotation_metrics_filter\" + {\n\t\ttargets = discovery.kubernetes.annotation_metrics.targets\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Targets to Keep or Drop\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare their metrics scraped or not\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/scrape: false\n\t\t//\n\t\t// the label prometheus.io/service-monitor: + \"false\" is a common label for headless services, when performing endpoint\n\t\t// + service discovery, if there is both a load-balanced service and headless service, + this can result in duplicate\n\t\t// scrapes if the name of the service is attached + as a label. any targets with this label or annotation set should be dropped\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"false\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, + \"endpoints\") + \"_label_prometheus_io_service_monitor\",\n\t\t\t]\n\t\t\tseparator + = \";\"\n\t\t\t// only allow empty or true, otherwise defaults to false\n\t\t\tregex + \ = \"^(?:;*)?(true)(;|true)*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\t// add a __tmp_scrape_port_named_metrics from + the argument.scrape_port_named_metrics\n\t\trule {\n\t\t\treplacement = format(\"%t\", + argument.scrape_port_named_metrics.value)\n\t\t\ttarget_label = \"__tmp_scrape_port_named_metrics\"\n\t\t}\n\n\t\t// + only keep targets that have scrape: true or \"metrics\" in the port name if the + argument scrape_port_named_metrics\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__tmp_scrape\",\n\t\t\t\t\"__tmp_scrape_port_named_metrics\",\n\t\t\t\t// + endpoints is the role and most meta labels started with \"endpoints\", however + the port name is an exception and starts with \"endpoint\"\n\t\t\t\t\"__meta_kubernetes_\" + + replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"endpoint\") + + \"_port_name\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(true;.*|(|true);true;(.*metrics.*))$\"\n\t\t}\n\n\t\t// + only keep targets where the pod is running or the pod_phase is empty and is not + an init container. This will only exist for role=\"pod\" or\n\t\t// potentially + role=\"endpoints\", if it is a service the value is empty and thus allowed to + pass, if it is an endpoint but not associated to a\n\t\t// pod but rather a static + IP or hostname, that could be outside of kubernetes allow endpoints to declare + what tenant their metrics should be\n\t\t// written to\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex + \ = \"^(?i)(Running|)$\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__meta_kubernetes_pod_ready\"]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\t\t// + if the container is an init container, drop it\n\t\trule {\n\t\t\taction = + \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_container_init\"]\n\t\t\tregex + \ = \"^(true)$\"\n\t\t}\n\n\t\t// allow resources to declare their metrics + the tenant their metrics should be sent to,\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/tenant: primary\n\t\t//\n\t\t// Note: This does not necessarily + have to be the actual tenantId, it can be a friendly name as well that is simply + used\n\t\t// to determine if the metrics should be gathered for the current + tenant\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_tenant\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_tenant\",\n\t\t\t]\n\t\t\tregex + = \"^(\" + coalesce(argument.tenant.value, \".*\") + \")$\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Scrape Metadata i.e. path, port, interval etc.\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare the protocol to use when collecting metrics, the default + value is \"http\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/scheme: + http\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"http\"\n\t\t\ttarget_label + = \"__scheme__\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scheme\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scheme\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?(https?).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scheme__\"\n\t\t}\n\n\t\t// allow resources + to declare the port to use when collecting metrics, the default value is the discovered + port from\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/port: 9090\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__address__\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_port\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_port\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^:]+)(?::\\\\d+)?;(\\\\d+)$\"\n\t\t\treplacement + \ = \"$1:$2\"\n\t\t\ttarget_label = \"__address__\"\n\t\t}\n\n\t\t// allow resources + to declare their the path to use when collecting their metrics, the default value + is \"/metrics\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/path: + /metrics/foo\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_path\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_path\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__metrics_path__\"\n\t\t}\n\n\t\t// allow resources + to declare how often their metrics should be collected, the default value is 1m,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/interval: 5m\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_interval\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_interval\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\t// allow + resources to declare the timeout of the scrape request, the default value is 10s,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/timeout: 30s\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_timeout\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_timeout\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Common Labels\n\t\t****************************************************************************************************************/\n\t\t// + set a source label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = \"kubernetes\"\n\t\t\ttarget_label = \"source\"\n\t\t}\n\n\t\t// set the cluster + label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = argument.cluster.value\n\t\t\ttarget_label + = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label + \ = \"namespace\"\n\t\t}\n\n\t\t// set the target name label i.e. service name, + pod name, etc.\n\t\t// if the role is endpoints, the first valued field is used + which would be __meta_kubernetes_pod_name, if the pod name is empty\n\t\t// then + the endpoint name would be used\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + argument.__pod_role.value + \"_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = + \"$1\"\n\t\t\ttarget_label = argument.__pod_role.value\n\t\t}\n\n\t\t// set a + default job label to be the namespace/pod_controller_name or namespace/service_name\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t\targument.__pod_role.value,\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^;]+)(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// if the controller + is a ReplicaSet, drop the hash from the end of the ReplicaSet\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_type\",\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:ReplicaSet);([^;]+);([^;]+)-.+$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// allow resources to + declare their the job label value to use when collecting their metrics, the default + value is \"\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/job: + integrations/kubernetes/cadvisor\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_job\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_job\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// set the app name if + specified as metadata labels \"app:\" or \"app.kubernetes.io/name:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"app\"\n\t\t}\n\n\t\t// + set the app component if specified as metadata labels \"component:\" or \"app.kubernetes.io/component:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_component\",\n\t\t\t]\n\t\t\tregex + \ = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"component\"\n\t\t}\n\n\t\t// set the version if specified as metadata labels + \"version:\" or \"app.kubernetes.io/version:\" or \"app_version:\"\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_version\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"version\"\n\t\t}\n\n\t\t// + set a workload label if the resource is a pod\n\t\t// example: grafana-agent-68nv9 + becomes DaemonSet/grafana-agent\n\t\trule {\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_kind\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"(.+);(.+)\"\n\t\t\treplacement = \"$1/$2\"\n\t\t\ttarget_label + = \"workload\"\n\t\t}\n\t\t// remove the hash from the ReplicaSet\n\t\trule {\n\t\t\tsource_labels + = [\"workload\"]\n\t\t\tregex = \"(ReplicaSet/.+)-.+\"\n\t\t\ttarget_label + \ = \"workload\"\n\t\t}\n\t}\n\n\t// only keep http targets\n\tdiscovery.relabel + \"http_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"http\"\n\t\t}\n\t}\n\n\t// only keep https targets\n\tdiscovery.relabel + \"https_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"https\"\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Scrape Labels Targets\n\t*****************************************************************/\n\t// + scrape http only targtetsa\n\tprometheus.scrape \"http_annotations\" {\n\t\ttargets + = discovery.relabel.http_annotations.output\n\n\t\tjob_name = \"annotation-metrics-http\"\n\t\tscheme + \ = \"http\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t// + scrape https only targtets\n\tprometheus.scrape \"https_annotations\" {\n\t\ttargets + = discovery.relabel.https_annotations.output\n\n\t\tjob_name = \"annotation-metrics-https\"\n\t\tscheme + \ = \"https\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\tbearer_token_file + = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n\n\t\ttls_config {\n\t\t\tca_file + \ = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n\t\t\tinsecure_skip_verify + = false\n\t\t\tserver_name = \"kubernetes\"\n\t\t}\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"annotations\" {\n\t\tforward_to = argument.forward_to.value\n\t\t// keep only + metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(.+)\")\n\t\t}\n\n\t\t// drop metrics that match the drop_metrics regex\n\t\trule + {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex + \ = coalesce(argument.drop_metrics.value, \"\")\n\t\t}\n\t}\n}\n" podmonitors-scrape.alloy: "/*\nModule Components: podmonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.podmonitors\n*/\n\ndeclare \"podmonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape PodMonitors\n\t*****************************************************************/\n\tprometheus.operator.podmonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.podmonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\t// selector {\n\t\t// \tmatch_expression {\n\t\t// \t\tkey = \"team\"\n\t\t// \t\toperator = \"In\"\n\t\t// \t\tvalues - \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n}\n" + \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"podmonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set the + cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" + rules-to-mimir.alloy: "/*\nModule Components: rules_to_mimir\nDescription: Auto + discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance.\n*/\n\ndeclare + \"rules_to_mimir\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + anonymous)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Mimir\n\t********************************************/\n\tmimir.rules.kubernetes + \"rules_to_mimir\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:8080\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\t// rule_selector {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= + \"true\",\n\t\t// \t}\n\t\t// }\n\t}\n}\n" servicemonitors-scrape.alloy: "/*\nModule Components: servicemonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.servicemonitors\n*/\n\ndeclare \"servicemonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape ServiceMonitors\n\t*****************************************************************/\n\tprometheus.operator.servicemonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.servicemonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering - {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n" + {\n\t\t\tenabled = true\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"servicemonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set + the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f namespace: monitoring-system --- apiVersion: v1 @@ -426,19 +773,21 @@ data: \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(ProfilessReceiver) where collected - logs should be forwarded to\"\n\t}\n\n\tdiscovery.kubernetes \"pyroscope_kubernetes\" - {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape config allows to define - annotations based scraping.\n\t//\n\t// For example the following annotations:\n\t//\n\t// - ```\n\t// profiles.grafana.com/memory.scrape: \"true\"\n\t// profiles.grafana.com/memory.port: - \"8080\"\n\t// profiles.grafana.com/cpu.scrape: \"true\"\n\t// profiles.grafana.com/cpu.port: - \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: \"true\"\n\t// profiles.grafana.com/goroutine.port: - \"8080\"\n\t// ```\n\t//\n\t// will scrape the `memory`, `cpu` and `goroutine` - profiles from the `8080` port of the pod.\n\t//\n\t// For more information see - https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel + logs should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\tdiscovery.kubernetes + \"pyroscope_kubernetes\" {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape + config allows to define annotations based scraping.\n\t//\n\t// For example the + following annotations:\n\t//\n\t// ```\n\t// profiles.grafana.com/memory.scrape: + \"true\"\n\t// profiles.grafana.com/memory.port: \"8080\"\n\t// profiles.grafana.com/cpu.scrape: + \"true\"\n\t// profiles.grafana.com/cpu.port: \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: + \"true\"\n\t// profiles.grafana.com/goroutine.port: \"8080\"\n\t// ```\n\t//\n\t// + will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port + of the pod.\n\t//\n\t// For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel \"kubernetes_pods\" {\n\t\ttargets = concat(discovery.kubernetes.pyroscope_kubernetes.targets)\n\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex \ = \"Pending|Succeeded|Failed|Completed\"\n\t\t}\n\n\t\trule {\n\t\t\taction - = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\trule + = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\trule @@ -657,7 +1006,7 @@ data: {\n\t\t\t\tenabled = true\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 namespace: monitoring-system --- apiVersion: v1 @@ -667,40 +1016,58 @@ data: Transformation\ndeclare \"process_and_transform\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"traces_forward_to\" {\n\t\tcomment = \"Must be a list(TracesReceiver) where - collected traces should be forwarded to\"\n\t}\n\n\targument \"logs_forward_to\" - {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be - forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment = \"Must - be a list(MetricsReceiver) where collected metrics should be forwarded to\"\n\t}\n\n\targument - \"cluster\" {\n\t\toptional = true\n\t\tdefault = \"k3d-k3s-codelab\"\n\t}\n\n\targument - \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4318\"\n\t}\n\n\targument - \"otlp_grpc_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* + collected traces should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\targument + \"logs_forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected + logs should be forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment + = \"Must be a list(MetricsReceiver) where collected metrics should be forwarded + to\"\n\t}\n\n\targument \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault + \ = \"0.0.0.0:4318\"\n\t}\n\n\targument \"otlp_grpc_endpoint\" {\n\t\toptional + = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* Jaeger for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.jaeger \"default\" {\n\t\tprotocols {\n\t\t\tgrpc {\n\t\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t\t}\n\n\t\t\tthrift_http {\n\t\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t\t}\n\n\t\t\tthrift_binary {\n\t\t\t\tendpoint = \"0.0.0.0:6832\"\n\t\t\t}\n\n\t\t\tthrift_compact {\n\t\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [otelcol.processor.resourcedetection.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + {\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* Otelcol for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.otlp - \"default\" {\n\t\tgrpc {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp - {\n\t\t\tendpoint = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics - = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection - \"default\" {\n\t\tdetectors = [\"env\"]\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces - = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes + \"default\" {\n\t\tdebug_metrics {\n\t\t\tdisable_high_cardinality_metrics = true\n\t\t}\n\n\t\tgrpc + {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp {\n\t\t\tendpoint + = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.resourcedetection.default.input]\n\t\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection + \"default\" {\n\t\tdetectors = [\"env\", \"system\"]\n\n\t\tsystem {\n\t\t\thostname_sources + = [\"os\"]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\n\t\t\tlogs + \ = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform + \"add_metric_datapoint_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements + {\n\t\t\tcontext = \"datapoint\"\n\t\t\tstatements = [\n\t\t\t\t\"set(attributes[\\\"deployment.environment\\\"], + resource.attributes[\\\"deployment.environment\\\"])\",\n\t\t\t\t\"set(attributes[\\\"service.version\\\"], + resource.attributes[\\\"service.version\\\"])\",\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes \"default\" {\n\t\textract {\n\t\t\tmetadata = [\n\t\t\t\t\"k8s.namespace.name\",\n\t\t\t\t\"k8s.pod.name\",\n\t\t\t\t\"k8s.deployment.name\",\n\t\t\t\t\"k8s.statefulset.name\",\n\t\t\t\t\"k8s.daemonset.name\",\n\t\t\t\t\"k8s.cronjob.name\",\n\t\t\t\t\"k8s.job.name\",\n\t\t\t\t\"k8s.node.name\",\n\t\t\t\t\"k8s.pod.uid\",\n\t\t\t\t\"k8s.pod.start_time\",\n\t\t\t]\n\t\t}\n\n\t\tpod_association {\n\t\t\tsource {\n\t\t\t\tfrom = \"connection\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t\ttraces - = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform - \"add_resource_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tlog_statements + {\n\t\t\tmetrics = [otelcol.processor.transform.default.input]\n\t\t\tlogs = + [otelcol.processor.transform.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.transform.default.input,\n\t\t\t\totelcol.connector.host_info.default.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.connector.host_info + \"default\" {\n\t\thost_identifiers = [\"k8s.node.name\"]\n\n\t\toutput {\n\t\t\tmetrics + = [otelcol.processor.batch.host_info_batch.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + \"host_info_batch\" {\n\t\toutput {\n\t\t\tmetrics = [otelcol.exporter.prometheus.host_info_metrics.input]\n\t\t}\n\t}\n\n\totelcol.exporter.prometheus + \"host_info_metrics\" {\n\t\tadd_metric_suffixes = false\n\t\tforward_to = + argument.metrics_forward_to.value\n\t}\n\n\totelcol.processor.transform \"default\" + {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements {\n\t\t\tcontext = + \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\tlog_statements {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])`,\n\t\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t\t`set(attributes[\"loki.resource.labels\"], \"pod, namespace, cluster, job\")`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements - {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], - \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter - \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.batch.default.input]\n\t\t\ttraces - = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements + {\n\t\t\tcontext = \"span\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces + \ = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter + \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\ttraces {\n\t\t\tspan = [\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/live\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] == \\\"/healthy\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/ready\\\"\",\n\t\t\t]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs + \ = [otelcol.processor.batch.default.input]\n\t\t\ttraces = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch \"default\" {\n\t\tsend_batch_size = 16384\n\t\tsend_batch_max_size = 0\n\t\ttimeout \ = \"5s\"\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\t\tlogs \ = [otelcol.processor.memory_limiter.default.input]\n\t\t\ttraces = [otelcol.processor.memory_limiter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.memory_limiter @@ -736,7 +1103,7 @@ data: \"alloy_traces_input\" {\n\t\tvalue = otelcol.processor.batch.default.input\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 namespace: monitoring-system --- apiVersion: v1 @@ -2342,19 +2709,19 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-t69dk2g68h + name: alloy-config-284mhm8bfc name: config - configMap: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f name: modules-kubernetes-metrics - configMap: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k name: modules-kubernetes-logs - configMap: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 name: modules-kubernetes-traces - configMap: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 name: modules-kubernetes-profiles --- apiVersion: apps/v1 diff --git a/kubernetes/microservices-mode/traces/traces.alloy b/kubernetes/microservices-mode/traces/traces.alloy index cbae15b5..42f255e1 100644 --- a/kubernetes/microservices-mode/traces/traces.alloy +++ b/kubernetes/microservices-mode/traces/traces.alloy @@ -28,6 +28,8 @@ import.file "traces" { // traces Processing And Transformation process_and_transform traces.process_and_transform "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + metrics_forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] logs_forward_to = [provider.self_hosted_stack.kubernetes.logs_receiver] traces_forward_to = [provider.self_hosted_stack.kubernetes.traces_receiver] @@ -48,10 +50,16 @@ import.file "metrics" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/metrics" } -metrics.podmonitors_scrape "kubernetes" { +metrics.annotations_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } metrics.servicemonitors_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } diff --git a/kubernetes/monolithic-mode/all-in-one/all-in-one.alloy b/kubernetes/monolithic-mode/all-in-one/all-in-one.alloy index b38f6905..294dfae9 100644 --- a/kubernetes/monolithic-mode/all-in-one/all-in-one.alloy +++ b/kubernetes/monolithic-mode/all-in-one/all-in-one.alloy @@ -27,11 +27,19 @@ import.file "metrics" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/metrics" } -metrics.podmonitors_scrape "kubernetes" { +metrics.rules_to_mimir "kubernetes" { } + +metrics.annotations_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } metrics.servicemonitors_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } @@ -42,9 +50,18 @@ import.file "logs" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/logs" } +logs.rules_to_loki "kubernetes" { } + +logs.kubernetes_cluster_events "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + + forward_to = [logs.keep_labels.kubernetes.receiver] +} + logs.annotations_scrape "kubernetes" { - forward_to = [logs.keep_labels.kubernetes.receiver] - annotation_prefix = "logs.grafana.com" + cluster = "k3d-k3s-codelab" + + forward_to = [logs.keep_labels.kubernetes.receiver] } logs.keep_labels "kubernetes" { @@ -60,6 +77,8 @@ import.file "traces" { // traces Processing And Transformation process_and_transform traces.process_and_transform "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + metrics_forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] logs_forward_to = [provider.self_hosted_stack.kubernetes.logs_receiver] traces_forward_to = [provider.self_hosted_stack.kubernetes.traces_receiver] @@ -81,8 +100,9 @@ import.file "profiles" { } profiles.annotations_scrape "kubernetes" { + cluster = "k3d-k3s-codelab" + forward_to = [provider.self_hosted_stack.kubernetes.profiles_receiver] - // annotation_prefix = "profiles.grafana.com" } /***************************************************************** @@ -99,8 +119,32 @@ import.string "memcached" { } memcached.memcached_metrics_scrape "instance" { - forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] - namespace = "monitoring-system" name = remote.kubernetes.configmap.integrations.data["MEMCACHED_K8S_SECRET_NAME"] + + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } + +// // Redis Integrations +// import.string "redis" { +// content = remote.kubernetes.configmap.integrations.data["redis.alloy"] +// } + +// redis.redis_exporter_metrics_scrape "instance" { +// namespace = "monitoring-system" +// name = remote.kubernetes.configmap.integrations.data["REDIS_K8S_SECRET_NAME"] + +// forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] +// } + +// // Mysql Integrations +// import.string "mysql" { +// content = remote.kubernetes.configmap.integrations.data["mysql.alloy"] +// } + +// mysql.mysql_metrics_scrape "instance" { +// namespace = "monitoring-system" +// name = remote.kubernetes.configmap.integrations.data["MYSQL_K8S_SECRET_NAME"] + +// forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] +// } diff --git a/kubernetes/monolithic-mode/all-in-one/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/all-in-one/k8s-all-in-one.yaml index 346a3044..376e7d84 100644 --- a/kubernetes/monolithic-mode/all-in-one/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/all-in-one/k8s-all-in-one.yaml @@ -631,35 +631,46 @@ data: = coalesce(env(\"SELF_HOSTED_PROFILES_ENDPOINT_URL\"), \"http://nginx.gateway.svc:4040\")\n}\n\n/********************************************\n * Metrics\n ********************************************/\nimport.file \"metrics\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/metrics\"\n}\n\nmetrics.podmonitors_scrape \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape - \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\n/********************************************\n + + \"/kubernetes/metrics\"\n}\n\nmetrics.rules_to_mimir \"kubernetes\" { }\n\nmetrics.annotations_scrape + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\n/********************************************\n * Logs\n ********************************************/\nimport.file \"logs\" {\n\tfilename - = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/logs\"\n}\n\nlogs.annotations_scrape - \"kubernetes\" {\n\tforward_to = [logs.keep_labels.kubernetes.receiver]\n\tannotation_prefix - = \"logs.grafana.com\"\n}\n\nlogs.keep_labels \"kubernetes\" {\n\tforward_to = - [provider.self_hosted_stack.kubernetes.logs_receiver]\n}\n\n/********************************************\n + = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/logs\"\n}\n\nlogs.rules_to_loki + \"kubernetes\" { }\n\nlogs.kubernetes_cluster_events \"kubernetes\" {\n\tcluster + = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\n\tforward_to = [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.annotations_scrape + \"kubernetes\" {\n\tcluster = \"k3d-k3s-codelab\"\n\n\tforward_to = [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.keep_labels + \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.logs_receiver]\n}\n\n/********************************************\n * Traces\n ********************************************/\nimport.file \"traces\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/traces\"\n}\n\n// traces Processing And Transformation process_and_transform\ntraces.process_and_transform - \"kubernetes\" {\n\tmetrics_forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\tlogs_forward_to + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\n\tmetrics_forward_to + = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\tlogs_forward_to \ = [provider.self_hosted_stack.kubernetes.logs_receiver]\n\ttraces_forward_to \ = [provider.self_hosted_stack.kubernetes.traces_receiver]\n}\n\ntracing {\n\t// Write all spans. Don't do this in production!\n\tsampling_fraction = 1\n\n\t// Forward Alloy internal spans to traces process.\n\twrite_to = [traces.process_and_transform.kubernetes.alloy_traces_input]\n}\n\n/********************************************\n * Profiles\n ********************************************/\nimport.file \"profiles\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/profiles\"\n}\n\nprofiles.annotations_scrape \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.profiles_receiver]\n\t// annotation_prefix - = \"profiles.grafana.com\"\n}\n\n/*****************************************************************\n* + + \"/kubernetes/profiles\"\n}\n\nprofiles.annotations_scrape \"kubernetes\" {\n\tcluster + = \"k3d-k3s-codelab\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.profiles_receiver]\n}\n\n/*****************************************************************\n* Alloy Integrations\n*****************************************************************/\nremote.kubernetes.configmap \"integrations\" {\n\tnamespace = \"monitoring-system\"\n\tname = \"alloy-integrations\"\n}\n\n// Memcached Integrations\nimport.string \"memcached\" {\n\tcontent = remote.kubernetes.configmap.integrations.data[\"memcached.alloy\"]\n}\n\nmemcached.memcached_metrics_scrape - \"instance\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\n\tnamespace - = \"monitoring-system\"\n\tname = remote.kubernetes.configmap.integrations.data[\"MEMCACHED_K8S_SECRET_NAME\"]\n}\n" + \"instance\" {\n\tnamespace = \"monitoring-system\"\n\tname = remote.kubernetes.configmap.integrations.data[\"MEMCACHED_K8S_SECRET_NAME\"]\n\n\tforward_to + = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\n// // Redis Integrations\n// + import.string \"redis\" {\n// \tcontent = remote.kubernetes.configmap.integrations.data[\"redis.alloy\"]\n// + }\n\n// redis.redis_exporter_metrics_scrape \"instance\" {\n// \tnamespace = \"monitoring-system\"\n// + \tname = remote.kubernetes.configmap.integrations.data[\"REDIS_K8S_SECRET_NAME\"]\n\n// + \tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n// }\n\n// + // Mysql Integrations\n// import.string \"mysql\" {\n// \tcontent = remote.kubernetes.configmap.integrations.data[\"mysql.alloy\"]\n// + }\n\n// mysql.mysql_metrics_scrape \"instance\" {\n// \tnamespace = \"monitoring-system\"\n// + \tname = remote.kubernetes.configmap.integrations.data[\"MYSQL_K8S_SECRET_NAME\"]\n\n// + \tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n// }\n" kind: ConfigMap metadata: - name: alloy-config-tdm88h78mg + name: alloy-config-552f8d68dg namespace: monitoring-system --- apiVersion: v1 @@ -784,28 +795,25 @@ apiVersion: v1 data: annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: Scrapes targets for logs based on kubernetes Pod annotations\n\n Annotations:\n - \ logs.grafana.com/ingest: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare + \ logs.grafana.com/scrape: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this - can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\t// arguments - for kubernetes discovery\n\targument \"namespaces\" {\n\t\tcomment = \"The namespaces - to look for targets in (default: [\\\"kube-system\\\"] is all namespaces)\"\n\t\toptional - = true\n\t}\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\targument + \"cluster\" { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix to use (default: logs.grafana.com)\"\n\t\tdefault = \"logs.grafana.com\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional = true\n\t\tcomment \ = \"The logic is used to transform the annotation argument into a valid label name by removing unsupported characters.\"\n\t\tdefault = replace(replace(replace(coalesce(argument.annotation_prefix.value, \"logs.grafana.com\"), \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\t// - find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\n\t\tnamespaces - {\n\t\t\tnames = coalesce(argument.namespaces.value, [])\n\t\t}\n\t}\n\n\t// filter - logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" + find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\t}\n\n\t// + filter logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" {\n\t\ttargets = discovery.kubernetes.annotation_logs.targets\n\n\t\t// allow pods to declare their logs to be ingested or not, the default is true\n\t\t// - \ i.e. logs.grafana.com/ingest: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + \ i.e. logs.grafana.com/scrape: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\n\t\t// allow pods to declare what tenant their logs should be written to, the following annotation @@ -815,6 +823,8 @@ data: + argument.tenant.value + \")$\"\n\t\t}\n\n\t\t// set the instance label as the name of the worker node the pod is on\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_node_name\"]\n\t\t\ttarget_label = \"instance\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\t// set the pod label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\t// @@ -879,12 +889,11 @@ data: ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\toptional - = true\n\t}\n\n\targument \"cluster\" {\n\t\toptional = true\n\t}\n\n\tloki.source.kubernetes_events - \"cluster_events\" {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format + = true\n\t}\n\n\targument \"cluster\" { }\n\n\tloki.source.kubernetes_events \"cluster_events\" + {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format = \"logfmt\"\n\t\tforward_to = [loki.process.logs_service.receiver]\n\t}\n\n\tloki.process \"logs_service\" {\n\t\tstage.static_labels {\n\t\t\tvalues = {\n\t\t\t\tcluster - = coalesce(argument.cluster.value, \"k3d\"),\n\t\t\t}\n\t\t}\n\t\tforward_to = - argument.forward_to.value\n\t}\n}\n" + = argument.cluster.value,\n\t\t\t}\n\t\t}\n\t\tforward_to = argument.forward_to.value\n\t}\n}\n" keep-labels.alloy: "/*\nModule Components: keep_labels\nDescription: Pre-defined set of labels to keep, this stage should always be in-place as the previous relabeing\n \ stages make every pod label and annotation a label in the pipeline, @@ -906,46 +915,392 @@ data: = argument.keep_labels.value\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* EXPORTS\n\t*****************************************************************/\n\texport \"receiver\" {\n\t\tvalue = loki.process.keep_labels.receiver\n\t}\n}\n" + rules-to-loki.alloy: "/*\nModule Components: rules_to_loki\nDescription: Auto discovers + PrometheusRule Kubernetes resources and loads them into a Loki instance.\n*/\n\ndeclare + \"rules_to_loki\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + fake)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Loki\n\t********************************************/\n\tloki.rules.kubernetes + \"rules_to_loki\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:3100\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_loki= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\trule_selector {\n\t\t\tmatch_labels = {\n\t\t\t\tauto_rules_to_loki + = \"true\",\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k namespace: monitoring-system --- apiVersion: v1 data: + annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: + Scrapes targets for metrics based on kubernetes annotations\n\nNote: Every argument + except for \"forward_to\" is optional, and does have a defined default value. + \ However, the values for these\n arguments are not defined using the default + = \" ... \" argument syntax, but rather using the coalesce(argument.value, \" + ... \").\n This is because if the argument passed in from another consuming + module is set to null, the default = \" ... \" syntax will\n does not override + the value passed in, where coalesce() will return the first non-null value.\n\nKubernetes + Annotation Auto-Scraping\n------------------------------------------------------------------------------------------------------------------------------------\nThis + module is meant to be used to automatically scrape targets based on a certain + role and set of annotations. This module can be consumed\nmultiple times with + different roles. The supported roles are:\n\n - pod\n - service\n - endpoints\n\nTypically, + if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality + you would use role=\"endpoints\", if\nmimicking the behavior of the PodMonitor + functionality you would use role=\"pod\". It is important to understand that + with endpoints,\nthe target is typically going to be a pod, and whatever annotations + that are set on the service will automatically be propagated to the\nendpoints. + \ This is why the role \"endpoints\" is used, because it will scrape the pod, + but also consider the service annotations. Using\nrole=\"endpoints\", which scrape + each endpoint associated to the service. If role=\"service\" is used, it will + only scrape the service, only\nhitting one of the endpoints associated to the + service.\n\nThis is where you must consider your scraping strategy, for example + if you scrape a service like \"kube-state-metrics\" using\nrole=\"endpoints\" + you should only have a single replica of the kube-state-metrics pod, if you have + multiple replicas, you should use\nrole=\"service\" or a separate non-annotation + job completely. Scraping a service instead of endpoints, is typically a rare + use case, but\nit is supported.\n\nThere are other considerations for using annotation + based scraping as well, which is metric relabeling rules that happen post scrape. + \ If\nyou have a target that you want to apply a bunch of relabelings to or a + very large metrics response payload, performance wise it will be\nbetter to have + a separate job for that target, rather than using use annotations. As every targert + will go through the ssame relabeling.\nTypical deployment strategies/options would + be:\n\nOption #1 (recommended):\n - Annotation Scraping for role=\"endpoints\"\n + \ - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, + etc.) or large metric payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, + kube-apiserver, kube-scheduler, etc.)\n\nOption #2:\n - Annotation Scraping for + role=\"pod\"\n - Annotation Scraping for role=\"service\" (i.e. kube-state-metrics, + node-exporter, etc.)\n - Separate Jobs for specific use cases or large metric + payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, + kube-scheduler, etc.)\n\nAt no point should you use role=\"endpoints\" and role=\"pod\" + together, as this will result in duplicate targets being scraped, thus\ngenerating + duplicate metrics. If you want to scrape both the pod and the service, use Option + #2.\n\nEach port attached to an service/pod/endpoint is an eligible target, oftentimes + it will have multiple ports.\nThere may be instances when you want to scrape all + ports or some ports and not others. To support this\nthe following annotations + are available:\n\n metrics.grafana.com/scrape: true\n\nthe default scraping scheme + is http, this can be specified as a single value which would override, the schema + being used for all\nports attached to the target:\n\n metrics.grafana.com/scheme: + https\n\nthe default path to scrape is /metrics, this can be specified as a single + value which would override, the scrape path being used\nfor all ports attached + to the target:\n\n metrics.grafana.com/path: /metrics/some_path\n\nthe default + port to scrape is the target port, this can be specified as a single value which + would override the scrape port being\nused for all ports attached to the target, + note that even if aan target had multiple targets, the relabel_config targets + are\ndeduped before scraping:\n\n metrics.grafana.com/port: 8080\n\nthe default + interval to scrape is 1m, this can be specified as a single value which would + override, the scrape interval being used\nfor all ports attached to the target:\n\n + \ metrics.grafana.com/interval: 5m\n\nthe default timeout for scraping is 10s, + this can be specified as a single value which would override, the scrape interval + being\nused for all ports attached to the target:\n\n metrics.grafana.com/timeout: + 30s\n\nthe default job is namespace/{{ service name }} or namespace/{{ controller_name + }} depending on the role, there may be instances\nin which a different job name + is required because of a set of dashboards, rules, etc. to support this there + is a job annotation\nwhich will override the default value:\n\n metrics.grafana.com/job: + integrations/kubernetes/kube-state-metrics\n*/\n\ndeclare \"annotations_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment + \ = \"The tenant to filter metrics to. This does not have to be the tenantId, + this is the value to look for in the metrics.agent.grafana.com/tenant annotation, + and this can be a regex.\"\n\t\toptional = true\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + to use (default: metrics.grafana.com)\"\n\t\tdefault = \"metrics.grafana.com\"\n\t\toptional + = true\n\t}\n\n\targument \"role\" {\n\t\tcomment = \"The role to use when looking + for targets to scrape via annotations, can be: endpoints, service, pod (default: + endpoints)\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional + = true\n\t\tcomment = \"The logic is used to transform the annotation argument + into a valid label name by removing unsupported characters.\"\n\t\tdefault = + replace(replace(replace(coalesce(argument.annotation_prefix.value, \"metrics.grafana.com\"), + \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\targument \"__pod_role\" + {\n\t\tcomment = \"Most annotation targets service or pod that is all you want, + however if the role is endpoints you want the pod\"\n\t\toptional = true\n\t\tdefault + \ = replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"pod\")\n\t}\n\n\targument + \"__service_role\" {\n\t\tcomment = \"Most annotation targets service or pod + that is all you want, however if the role is endpoints you we also want to consider + service annotations\"\n\t\toptional = true\n\t\tdefault = replace(coalesce(argument.role.value, + \"endpoints\"), \"endpoints\", \"service\")\n\t}\n\n\targument \"scrape_port_named_metrics\" + {\n\t\tcomment = \"Whether or not to automatically scrape endpoints that have + a port with 'metrics' in the name\"\n\t\toptional = true\n\t\tdefault = false\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: (.+))\"\n\t\toptional + = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment = \"A regex of metrics + to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" + {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + Targets From Docker Discovery\n\t*****************************************************************/\n\tdiscovery.kubernetes + \"annotation_metrics\" {\n\t\trole = coalesce(argument.role.value, \"endpoints\")\n\n\t\tselectors + {\n\t\t\trole = coalesce(argument.role.value, \"endpoints\")\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t*****************************************************************/\n\t// + filter metrics by kubernetes annotations\n\tdiscovery.relabel \"annotation_metrics_filter\" + {\n\t\ttargets = discovery.kubernetes.annotation_metrics.targets\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Targets to Keep or Drop\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare their metrics scraped or not\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/scrape: false\n\t\t//\n\t\t// the label prometheus.io/service-monitor: + \"false\" is a common label for headless services, when performing endpoint\n\t\t// + service discovery, if there is both a load-balanced service and headless service, + this can result in duplicate\n\t\t// scrapes if the name of the service is attached + as a label. any targets with this label or annotation set should be dropped\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"false\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, + \"endpoints\") + \"_label_prometheus_io_service_monitor\",\n\t\t\t]\n\t\t\tseparator + = \";\"\n\t\t\t// only allow empty or true, otherwise defaults to false\n\t\t\tregex + \ = \"^(?:;*)?(true)(;|true)*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\t// add a __tmp_scrape_port_named_metrics from + the argument.scrape_port_named_metrics\n\t\trule {\n\t\t\treplacement = format(\"%t\", + argument.scrape_port_named_metrics.value)\n\t\t\ttarget_label = \"__tmp_scrape_port_named_metrics\"\n\t\t}\n\n\t\t// + only keep targets that have scrape: true or \"metrics\" in the port name if the + argument scrape_port_named_metrics\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__tmp_scrape\",\n\t\t\t\t\"__tmp_scrape_port_named_metrics\",\n\t\t\t\t// + endpoints is the role and most meta labels started with \"endpoints\", however + the port name is an exception and starts with \"endpoint\"\n\t\t\t\t\"__meta_kubernetes_\" + + replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"endpoint\") + + \"_port_name\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(true;.*|(|true);true;(.*metrics.*))$\"\n\t\t}\n\n\t\t// + only keep targets where the pod is running or the pod_phase is empty and is not + an init container. This will only exist for role=\"pod\" or\n\t\t// potentially + role=\"endpoints\", if it is a service the value is empty and thus allowed to + pass, if it is an endpoint but not associated to a\n\t\t// pod but rather a static + IP or hostname, that could be outside of kubernetes allow endpoints to declare + what tenant their metrics should be\n\t\t// written to\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex + \ = \"^(?i)(Running|)$\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__meta_kubernetes_pod_ready\"]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\t\t// + if the container is an init container, drop it\n\t\trule {\n\t\t\taction = + \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_container_init\"]\n\t\t\tregex + \ = \"^(true)$\"\n\t\t}\n\n\t\t// allow resources to declare their metrics + the tenant their metrics should be sent to,\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/tenant: primary\n\t\t//\n\t\t// Note: This does not necessarily + have to be the actual tenantId, it can be a friendly name as well that is simply + used\n\t\t// to determine if the metrics should be gathered for the current + tenant\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_tenant\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_tenant\",\n\t\t\t]\n\t\t\tregex + = \"^(\" + coalesce(argument.tenant.value, \".*\") + \")$\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Scrape Metadata i.e. path, port, interval etc.\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare the protocol to use when collecting metrics, the default + value is \"http\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/scheme: + http\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"http\"\n\t\t\ttarget_label + = \"__scheme__\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scheme\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scheme\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?(https?).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scheme__\"\n\t\t}\n\n\t\t// allow resources + to declare the port to use when collecting metrics, the default value is the discovered + port from\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/port: 9090\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__address__\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_port\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_port\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^:]+)(?::\\\\d+)?;(\\\\d+)$\"\n\t\t\treplacement + \ = \"$1:$2\"\n\t\t\ttarget_label = \"__address__\"\n\t\t}\n\n\t\t// allow resources + to declare their the path to use when collecting their metrics, the default value + is \"/metrics\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/path: + /metrics/foo\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_path\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_path\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__metrics_path__\"\n\t\t}\n\n\t\t// allow resources + to declare how often their metrics should be collected, the default value is 1m,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/interval: 5m\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_interval\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_interval\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\t// allow + resources to declare the timeout of the scrape request, the default value is 10s,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/timeout: 30s\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_timeout\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_timeout\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Common Labels\n\t\t****************************************************************************************************************/\n\t\t// + set a source label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = \"kubernetes\"\n\t\t\ttarget_label = \"source\"\n\t\t}\n\n\t\t// set the cluster + label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = argument.cluster.value\n\t\t\ttarget_label + = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label + \ = \"namespace\"\n\t\t}\n\n\t\t// set the target name label i.e. service name, + pod name, etc.\n\t\t// if the role is endpoints, the first valued field is used + which would be __meta_kubernetes_pod_name, if the pod name is empty\n\t\t// then + the endpoint name would be used\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + argument.__pod_role.value + \"_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = + \"$1\"\n\t\t\ttarget_label = argument.__pod_role.value\n\t\t}\n\n\t\t// set a + default job label to be the namespace/pod_controller_name or namespace/service_name\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t\targument.__pod_role.value,\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^;]+)(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// if the controller + is a ReplicaSet, drop the hash from the end of the ReplicaSet\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_type\",\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:ReplicaSet);([^;]+);([^;]+)-.+$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// allow resources to + declare their the job label value to use when collecting their metrics, the default + value is \"\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/job: + integrations/kubernetes/cadvisor\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_job\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_job\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// set the app name if + specified as metadata labels \"app:\" or \"app.kubernetes.io/name:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"app\"\n\t\t}\n\n\t\t// + set the app component if specified as metadata labels \"component:\" or \"app.kubernetes.io/component:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_component\",\n\t\t\t]\n\t\t\tregex + \ = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"component\"\n\t\t}\n\n\t\t// set the version if specified as metadata labels + \"version:\" or \"app.kubernetes.io/version:\" or \"app_version:\"\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_version\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"version\"\n\t\t}\n\n\t\t// + set a workload label if the resource is a pod\n\t\t// example: grafana-agent-68nv9 + becomes DaemonSet/grafana-agent\n\t\trule {\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_kind\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"(.+);(.+)\"\n\t\t\treplacement = \"$1/$2\"\n\t\t\ttarget_label + = \"workload\"\n\t\t}\n\t\t// remove the hash from the ReplicaSet\n\t\trule {\n\t\t\tsource_labels + = [\"workload\"]\n\t\t\tregex = \"(ReplicaSet/.+)-.+\"\n\t\t\ttarget_label + \ = \"workload\"\n\t\t}\n\t}\n\n\t// only keep http targets\n\tdiscovery.relabel + \"http_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"http\"\n\t\t}\n\t}\n\n\t// only keep https targets\n\tdiscovery.relabel + \"https_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"https\"\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Scrape Labels Targets\n\t*****************************************************************/\n\t// + scrape http only targtetsa\n\tprometheus.scrape \"http_annotations\" {\n\t\ttargets + = discovery.relabel.http_annotations.output\n\n\t\tjob_name = \"annotation-metrics-http\"\n\t\tscheme + \ = \"http\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t// + scrape https only targtets\n\tprometheus.scrape \"https_annotations\" {\n\t\ttargets + = discovery.relabel.https_annotations.output\n\n\t\tjob_name = \"annotation-metrics-https\"\n\t\tscheme + \ = \"https\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\tbearer_token_file + = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n\n\t\ttls_config {\n\t\t\tca_file + \ = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n\t\t\tinsecure_skip_verify + = false\n\t\t\tserver_name = \"kubernetes\"\n\t\t}\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"annotations\" {\n\t\tforward_to = argument.forward_to.value\n\t\t// keep only + metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(.+)\")\n\t\t}\n\n\t\t// drop metrics that match the drop_metrics regex\n\t\trule + {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex + \ = coalesce(argument.drop_metrics.value, \"\")\n\t\t}\n\t}\n}\n" podmonitors-scrape.alloy: "/*\nModule Components: podmonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.podmonitors\n*/\n\ndeclare \"podmonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape PodMonitors\n\t*****************************************************************/\n\tprometheus.operator.podmonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.podmonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\t// selector {\n\t\t// \tmatch_expression {\n\t\t// \t\tkey = \"team\"\n\t\t// \t\toperator = \"In\"\n\t\t// \t\tvalues - \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n}\n" + \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"podmonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set the + cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" + rules-to-mimir.alloy: "/*\nModule Components: rules_to_mimir\nDescription: Auto + discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance.\n*/\n\ndeclare + \"rules_to_mimir\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + anonymous)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Mimir\n\t********************************************/\n\tmimir.rules.kubernetes + \"rules_to_mimir\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:8080\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\t// rule_selector {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= + \"true\",\n\t\t// \t}\n\t\t// }\n\t}\n}\n" servicemonitors-scrape.alloy: "/*\nModule Components: servicemonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.servicemonitors\n*/\n\ndeclare \"servicemonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape ServiceMonitors\n\t*****************************************************************/\n\tprometheus.operator.servicemonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.servicemonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering - {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n" + {\n\t\t\tenabled = true\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"servicemonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set + the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f namespace: monitoring-system --- apiVersion: v1 @@ -955,19 +1310,21 @@ data: \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(ProfilessReceiver) where collected - logs should be forwarded to\"\n\t}\n\n\tdiscovery.kubernetes \"pyroscope_kubernetes\" - {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape config allows to define - annotations based scraping.\n\t//\n\t// For example the following annotations:\n\t//\n\t// - ```\n\t// profiles.grafana.com/memory.scrape: \"true\"\n\t// profiles.grafana.com/memory.port: - \"8080\"\n\t// profiles.grafana.com/cpu.scrape: \"true\"\n\t// profiles.grafana.com/cpu.port: - \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: \"true\"\n\t// profiles.grafana.com/goroutine.port: - \"8080\"\n\t// ```\n\t//\n\t// will scrape the `memory`, `cpu` and `goroutine` - profiles from the `8080` port of the pod.\n\t//\n\t// For more information see - https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel + logs should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\tdiscovery.kubernetes + \"pyroscope_kubernetes\" {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape + config allows to define annotations based scraping.\n\t//\n\t// For example the + following annotations:\n\t//\n\t// ```\n\t// profiles.grafana.com/memory.scrape: + \"true\"\n\t// profiles.grafana.com/memory.port: \"8080\"\n\t// profiles.grafana.com/cpu.scrape: + \"true\"\n\t// profiles.grafana.com/cpu.port: \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: + \"true\"\n\t// profiles.grafana.com/goroutine.port: \"8080\"\n\t// ```\n\t//\n\t// + will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port + of the pod.\n\t//\n\t// For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel \"kubernetes_pods\" {\n\t\ttargets = concat(discovery.kubernetes.pyroscope_kubernetes.targets)\n\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex \ = \"Pending|Succeeded|Failed|Completed\"\n\t\t}\n\n\t\trule {\n\t\t\taction - = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\trule + = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\trule @@ -1186,7 +1543,7 @@ data: {\n\t\t\t\tenabled = true\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 namespace: monitoring-system --- apiVersion: v1 @@ -1196,40 +1553,58 @@ data: Transformation\ndeclare \"process_and_transform\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"traces_forward_to\" {\n\t\tcomment = \"Must be a list(TracesReceiver) where - collected traces should be forwarded to\"\n\t}\n\n\targument \"logs_forward_to\" - {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be - forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment = \"Must - be a list(MetricsReceiver) where collected metrics should be forwarded to\"\n\t}\n\n\targument - \"cluster\" {\n\t\toptional = true\n\t\tdefault = \"k3d-k3s-codelab\"\n\t}\n\n\targument - \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4318\"\n\t}\n\n\targument - \"otlp_grpc_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* + collected traces should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\targument + \"logs_forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected + logs should be forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment + = \"Must be a list(MetricsReceiver) where collected metrics should be forwarded + to\"\n\t}\n\n\targument \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault + \ = \"0.0.0.0:4318\"\n\t}\n\n\targument \"otlp_grpc_endpoint\" {\n\t\toptional + = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* Jaeger for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.jaeger \"default\" {\n\t\tprotocols {\n\t\t\tgrpc {\n\t\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t\t}\n\n\t\t\tthrift_http {\n\t\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t\t}\n\n\t\t\tthrift_binary {\n\t\t\t\tendpoint = \"0.0.0.0:6832\"\n\t\t\t}\n\n\t\t\tthrift_compact {\n\t\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [otelcol.processor.resourcedetection.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + {\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* Otelcol for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.otlp - \"default\" {\n\t\tgrpc {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp - {\n\t\t\tendpoint = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics - = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection - \"default\" {\n\t\tdetectors = [\"env\"]\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces - = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes + \"default\" {\n\t\tdebug_metrics {\n\t\t\tdisable_high_cardinality_metrics = true\n\t\t}\n\n\t\tgrpc + {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp {\n\t\t\tendpoint + = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.resourcedetection.default.input]\n\t\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection + \"default\" {\n\t\tdetectors = [\"env\", \"system\"]\n\n\t\tsystem {\n\t\t\thostname_sources + = [\"os\"]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\n\t\t\tlogs + \ = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform + \"add_metric_datapoint_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements + {\n\t\t\tcontext = \"datapoint\"\n\t\t\tstatements = [\n\t\t\t\t\"set(attributes[\\\"deployment.environment\\\"], + resource.attributes[\\\"deployment.environment\\\"])\",\n\t\t\t\t\"set(attributes[\\\"service.version\\\"], + resource.attributes[\\\"service.version\\\"])\",\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes \"default\" {\n\t\textract {\n\t\t\tmetadata = [\n\t\t\t\t\"k8s.namespace.name\",\n\t\t\t\t\"k8s.pod.name\",\n\t\t\t\t\"k8s.deployment.name\",\n\t\t\t\t\"k8s.statefulset.name\",\n\t\t\t\t\"k8s.daemonset.name\",\n\t\t\t\t\"k8s.cronjob.name\",\n\t\t\t\t\"k8s.job.name\",\n\t\t\t\t\"k8s.node.name\",\n\t\t\t\t\"k8s.pod.uid\",\n\t\t\t\t\"k8s.pod.start_time\",\n\t\t\t]\n\t\t}\n\n\t\tpod_association {\n\t\t\tsource {\n\t\t\t\tfrom = \"connection\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t\ttraces - = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform - \"add_resource_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tlog_statements + {\n\t\t\tmetrics = [otelcol.processor.transform.default.input]\n\t\t\tlogs = + [otelcol.processor.transform.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.transform.default.input,\n\t\t\t\totelcol.connector.host_info.default.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.connector.host_info + \"default\" {\n\t\thost_identifiers = [\"k8s.node.name\"]\n\n\t\toutput {\n\t\t\tmetrics + = [otelcol.processor.batch.host_info_batch.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + \"host_info_batch\" {\n\t\toutput {\n\t\t\tmetrics = [otelcol.exporter.prometheus.host_info_metrics.input]\n\t\t}\n\t}\n\n\totelcol.exporter.prometheus + \"host_info_metrics\" {\n\t\tadd_metric_suffixes = false\n\t\tforward_to = + argument.metrics_forward_to.value\n\t}\n\n\totelcol.processor.transform \"default\" + {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements {\n\t\t\tcontext = + \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\tlog_statements {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])`,\n\t\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t\t`set(attributes[\"loki.resource.labels\"], \"pod, namespace, cluster, job\")`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements - {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], - \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter - \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.batch.default.input]\n\t\t\ttraces - = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements + {\n\t\t\tcontext = \"span\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces + \ = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter + \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\ttraces {\n\t\t\tspan = [\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/live\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] == \\\"/healthy\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/ready\\\"\",\n\t\t\t]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs + \ = [otelcol.processor.batch.default.input]\n\t\t\ttraces = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch \"default\" {\n\t\tsend_batch_size = 16384\n\t\tsend_batch_max_size = 0\n\t\ttimeout \ = \"5s\"\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\t\tlogs \ = [otelcol.processor.memory_limiter.default.input]\n\t\t\ttraces = [otelcol.processor.memory_limiter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.memory_limiter @@ -1265,7 +1640,7 @@ data: \"alloy_traces_input\" {\n\t\tvalue = otelcol.processor.batch.default.input\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 namespace: monitoring-system --- apiVersion: v1 @@ -2332,19 +2707,19 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-tdm88h78mg + name: alloy-config-552f8d68dg name: config - configMap: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f name: modules-kubernetes-metrics - configMap: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k name: modules-kubernetes-logs - configMap: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 name: modules-kubernetes-traces - configMap: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 name: modules-kubernetes-profiles --- apiVersion: apps/v1 diff --git a/kubernetes/monolithic-mode/logs/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/logs/k8s-all-in-one.yaml index 118455fb..e123246d 100644 --- a/kubernetes/monolithic-mode/logs/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/logs/k8s-all-in-one.yaml @@ -367,31 +367,26 @@ data: \ = coalesce(env(\"SELF_HOSTED_LOGS_ENDPOINT_URL\"), \"http://nginx.gateway.svc:3100/loki/api/v1/push\")\n}\n\n/********************************************\n * Logs\n ********************************************/\nimport.file \"logs\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/logs\"\n}\n\nlogs.kubernetes_cluster_events - \"kubernetes\" {\n\tcluster = \"k3d-k3s-codelab\"\n\tforward_to = [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.annotations_scrape - \"kubernetes\" {\n\tannotation_prefix = \"logs.grafana.com\"\n\tforward_to = - [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.keep_labels \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.logs_receiver]\n}\n\n/********************************************\n + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\n\tforward_to + = [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.annotations_scrape \"kubernetes\" + {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\n\tforward_to + = [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.keep_labels \"kubernetes\" + {\n\tforward_to = [provider.self_hosted_stack.kubernetes.logs_receiver]\n}\n\n/********************************************\n * Metrics\n ********************************************/\nimport.file \"metrics\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/metrics\"\n}\n\nmetrics.podmonitors_scrape \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape - \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\n/*****************************************************************\n* + + \"/kubernetes/metrics\"\n}\n\nmetrics.annotations_scrape \"kubernetes\" {\n\tcluster + \ = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\n/*****************************************************************\n* Alloy Integrations\n*****************************************************************/\nremote.kubernetes.configmap \"integrations\" {\n\tnamespace = \"monitoring-system\"\n\tname = \"alloy-integrations\"\n}\n\n// Memcached Integrations\nimport.string \"memcached\" {\n\tcontent = remote.kubernetes.configmap.integrations.data[\"memcached.alloy\"]\n}\n\nmemcached.memcached_metrics_scrape - \"instance\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\n\tnamespace - = \"monitoring-system\"\n\tname = remote.kubernetes.configmap.integrations.data[\"MEMCACHED_K8S_SECRET_NAME\"]\n}\n\n// - // Redis Integrations\n// import.string \"redis\" {\n// \tcontent = remote.kubernetes.configmap.integrations.data[\"redis.alloy\"]\n// - }\n\n// redis.redis_exporter_metrics_scrape \"instance\" {\n// \tforward_to = - [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\n// \tnamespace = \"monitoring-system\"\n// - \tname = remote.kubernetes.configmap.integrations.data[\"REDIS_K8S_SECRET_NAME\"]\n// - }\n\n// // Mysql Integrations\n// import.string \"mysql\" {\n// \tcontent = remote.kubernetes.configmap.integrations.data[\"mysql.alloy\"]\n// - }\n\n// mysql.mysql_metrics_scrape \"instance\" {\n// \tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\n// - \tnamespace = \"monitoring-system\"\n// \tname = remote.kubernetes.configmap.integrations.data[\"MYSQL_K8S_SECRET_NAME\"]\n// - }\n" + \"instance\" {\n\tnamespace = \"monitoring-system\"\n\tname = remote.kubernetes.configmap.integrations.data[\"MEMCACHED_K8S_SECRET_NAME\"]\n\n\tforward_to + = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-4m6599mmm2 + name: alloy-config-fmt85fgk68 namespace: monitoring-system --- apiVersion: v1 @@ -516,28 +511,25 @@ apiVersion: v1 data: annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: Scrapes targets for logs based on kubernetes Pod annotations\n\n Annotations:\n - \ logs.grafana.com/ingest: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare + \ logs.grafana.com/scrape: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this - can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\t// arguments - for kubernetes discovery\n\targument \"namespaces\" {\n\t\tcomment = \"The namespaces - to look for targets in (default: [\\\"kube-system\\\"] is all namespaces)\"\n\t\toptional - = true\n\t}\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\targument + \"cluster\" { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix to use (default: logs.grafana.com)\"\n\t\tdefault = \"logs.grafana.com\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional = true\n\t\tcomment \ = \"The logic is used to transform the annotation argument into a valid label name by removing unsupported characters.\"\n\t\tdefault = replace(replace(replace(coalesce(argument.annotation_prefix.value, \"logs.grafana.com\"), \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\t// - find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\n\t\tnamespaces - {\n\t\t\tnames = coalesce(argument.namespaces.value, [])\n\t\t}\n\t}\n\n\t// filter - logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" + find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\t}\n\n\t// + filter logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" {\n\t\ttargets = discovery.kubernetes.annotation_logs.targets\n\n\t\t// allow pods to declare their logs to be ingested or not, the default is true\n\t\t// - \ i.e. logs.grafana.com/ingest: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + \ i.e. logs.grafana.com/scrape: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\n\t\t// allow pods to declare what tenant their logs should be written to, the following annotation @@ -547,6 +539,8 @@ data: + argument.tenant.value + \")$\"\n\t\t}\n\n\t\t// set the instance label as the name of the worker node the pod is on\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_node_name\"]\n\t\t\ttarget_label = \"instance\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\t// set the pod label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\t// @@ -611,12 +605,11 @@ data: ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\toptional - = true\n\t}\n\n\targument \"cluster\" {\n\t\toptional = true\n\t}\n\n\tloki.source.kubernetes_events - \"cluster_events\" {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format + = true\n\t}\n\n\targument \"cluster\" { }\n\n\tloki.source.kubernetes_events \"cluster_events\" + {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format = \"logfmt\"\n\t\tforward_to = [loki.process.logs_service.receiver]\n\t}\n\n\tloki.process \"logs_service\" {\n\t\tstage.static_labels {\n\t\t\tvalues = {\n\t\t\t\tcluster - = coalesce(argument.cluster.value, \"k3d\"),\n\t\t\t}\n\t\t}\n\t\tforward_to = - argument.forward_to.value\n\t}\n}\n" + = argument.cluster.value,\n\t\t\t}\n\t\t}\n\t\tforward_to = argument.forward_to.value\n\t}\n}\n" keep-labels.alloy: "/*\nModule Components: keep_labels\nDescription: Pre-defined set of labels to keep, this stage should always be in-place as the previous relabeing\n \ stages make every pod label and annotation a label in the pipeline, @@ -638,46 +631,392 @@ data: = argument.keep_labels.value\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* EXPORTS\n\t*****************************************************************/\n\texport \"receiver\" {\n\t\tvalue = loki.process.keep_labels.receiver\n\t}\n}\n" + rules-to-loki.alloy: "/*\nModule Components: rules_to_loki\nDescription: Auto discovers + PrometheusRule Kubernetes resources and loads them into a Loki instance.\n*/\n\ndeclare + \"rules_to_loki\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + fake)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Loki\n\t********************************************/\n\tloki.rules.kubernetes + \"rules_to_loki\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:3100\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_loki= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\trule_selector {\n\t\t\tmatch_labels = {\n\t\t\t\tauto_rules_to_loki + = \"true\",\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k namespace: monitoring-system --- apiVersion: v1 data: + annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: + Scrapes targets for metrics based on kubernetes annotations\n\nNote: Every argument + except for \"forward_to\" is optional, and does have a defined default value. + \ However, the values for these\n arguments are not defined using the default + = \" ... \" argument syntax, but rather using the coalesce(argument.value, \" + ... \").\n This is because if the argument passed in from another consuming + module is set to null, the default = \" ... \" syntax will\n does not override + the value passed in, where coalesce() will return the first non-null value.\n\nKubernetes + Annotation Auto-Scraping\n------------------------------------------------------------------------------------------------------------------------------------\nThis + module is meant to be used to automatically scrape targets based on a certain + role and set of annotations. This module can be consumed\nmultiple times with + different roles. The supported roles are:\n\n - pod\n - service\n - endpoints\n\nTypically, + if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality + you would use role=\"endpoints\", if\nmimicking the behavior of the PodMonitor + functionality you would use role=\"pod\". It is important to understand that + with endpoints,\nthe target is typically going to be a pod, and whatever annotations + that are set on the service will automatically be propagated to the\nendpoints. + \ This is why the role \"endpoints\" is used, because it will scrape the pod, + but also consider the service annotations. Using\nrole=\"endpoints\", which scrape + each endpoint associated to the service. If role=\"service\" is used, it will + only scrape the service, only\nhitting one of the endpoints associated to the + service.\n\nThis is where you must consider your scraping strategy, for example + if you scrape a service like \"kube-state-metrics\" using\nrole=\"endpoints\" + you should only have a single replica of the kube-state-metrics pod, if you have + multiple replicas, you should use\nrole=\"service\" or a separate non-annotation + job completely. Scraping a service instead of endpoints, is typically a rare + use case, but\nit is supported.\n\nThere are other considerations for using annotation + based scraping as well, which is metric relabeling rules that happen post scrape. + \ If\nyou have a target that you want to apply a bunch of relabelings to or a + very large metrics response payload, performance wise it will be\nbetter to have + a separate job for that target, rather than using use annotations. As every targert + will go through the ssame relabeling.\nTypical deployment strategies/options would + be:\n\nOption #1 (recommended):\n - Annotation Scraping for role=\"endpoints\"\n + \ - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, + etc.) or large metric payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, + kube-apiserver, kube-scheduler, etc.)\n\nOption #2:\n - Annotation Scraping for + role=\"pod\"\n - Annotation Scraping for role=\"service\" (i.e. kube-state-metrics, + node-exporter, etc.)\n - Separate Jobs for specific use cases or large metric + payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, + kube-scheduler, etc.)\n\nAt no point should you use role=\"endpoints\" and role=\"pod\" + together, as this will result in duplicate targets being scraped, thus\ngenerating + duplicate metrics. If you want to scrape both the pod and the service, use Option + #2.\n\nEach port attached to an service/pod/endpoint is an eligible target, oftentimes + it will have multiple ports.\nThere may be instances when you want to scrape all + ports or some ports and not others. To support this\nthe following annotations + are available:\n\n metrics.grafana.com/scrape: true\n\nthe default scraping scheme + is http, this can be specified as a single value which would override, the schema + being used for all\nports attached to the target:\n\n metrics.grafana.com/scheme: + https\n\nthe default path to scrape is /metrics, this can be specified as a single + value which would override, the scrape path being used\nfor all ports attached + to the target:\n\n metrics.grafana.com/path: /metrics/some_path\n\nthe default + port to scrape is the target port, this can be specified as a single value which + would override the scrape port being\nused for all ports attached to the target, + note that even if aan target had multiple targets, the relabel_config targets + are\ndeduped before scraping:\n\n metrics.grafana.com/port: 8080\n\nthe default + interval to scrape is 1m, this can be specified as a single value which would + override, the scrape interval being used\nfor all ports attached to the target:\n\n + \ metrics.grafana.com/interval: 5m\n\nthe default timeout for scraping is 10s, + this can be specified as a single value which would override, the scrape interval + being\nused for all ports attached to the target:\n\n metrics.grafana.com/timeout: + 30s\n\nthe default job is namespace/{{ service name }} or namespace/{{ controller_name + }} depending on the role, there may be instances\nin which a different job name + is required because of a set of dashboards, rules, etc. to support this there + is a job annotation\nwhich will override the default value:\n\n metrics.grafana.com/job: + integrations/kubernetes/kube-state-metrics\n*/\n\ndeclare \"annotations_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment + \ = \"The tenant to filter metrics to. This does not have to be the tenantId, + this is the value to look for in the metrics.agent.grafana.com/tenant annotation, + and this can be a regex.\"\n\t\toptional = true\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + to use (default: metrics.grafana.com)\"\n\t\tdefault = \"metrics.grafana.com\"\n\t\toptional + = true\n\t}\n\n\targument \"role\" {\n\t\tcomment = \"The role to use when looking + for targets to scrape via annotations, can be: endpoints, service, pod (default: + endpoints)\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional + = true\n\t\tcomment = \"The logic is used to transform the annotation argument + into a valid label name by removing unsupported characters.\"\n\t\tdefault = + replace(replace(replace(coalesce(argument.annotation_prefix.value, \"metrics.grafana.com\"), + \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\targument \"__pod_role\" + {\n\t\tcomment = \"Most annotation targets service or pod that is all you want, + however if the role is endpoints you want the pod\"\n\t\toptional = true\n\t\tdefault + \ = replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"pod\")\n\t}\n\n\targument + \"__service_role\" {\n\t\tcomment = \"Most annotation targets service or pod + that is all you want, however if the role is endpoints you we also want to consider + service annotations\"\n\t\toptional = true\n\t\tdefault = replace(coalesce(argument.role.value, + \"endpoints\"), \"endpoints\", \"service\")\n\t}\n\n\targument \"scrape_port_named_metrics\" + {\n\t\tcomment = \"Whether or not to automatically scrape endpoints that have + a port with 'metrics' in the name\"\n\t\toptional = true\n\t\tdefault = false\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: (.+))\"\n\t\toptional + = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment = \"A regex of metrics + to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" + {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + Targets From Docker Discovery\n\t*****************************************************************/\n\tdiscovery.kubernetes + \"annotation_metrics\" {\n\t\trole = coalesce(argument.role.value, \"endpoints\")\n\n\t\tselectors + {\n\t\t\trole = coalesce(argument.role.value, \"endpoints\")\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t*****************************************************************/\n\t// + filter metrics by kubernetes annotations\n\tdiscovery.relabel \"annotation_metrics_filter\" + {\n\t\ttargets = discovery.kubernetes.annotation_metrics.targets\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Targets to Keep or Drop\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare their metrics scraped or not\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/scrape: false\n\t\t//\n\t\t// the label prometheus.io/service-monitor: + \"false\" is a common label for headless services, when performing endpoint\n\t\t// + service discovery, if there is both a load-balanced service and headless service, + this can result in duplicate\n\t\t// scrapes if the name of the service is attached + as a label. any targets with this label or annotation set should be dropped\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"false\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, + \"endpoints\") + \"_label_prometheus_io_service_monitor\",\n\t\t\t]\n\t\t\tseparator + = \";\"\n\t\t\t// only allow empty or true, otherwise defaults to false\n\t\t\tregex + \ = \"^(?:;*)?(true)(;|true)*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\t// add a __tmp_scrape_port_named_metrics from + the argument.scrape_port_named_metrics\n\t\trule {\n\t\t\treplacement = format(\"%t\", + argument.scrape_port_named_metrics.value)\n\t\t\ttarget_label = \"__tmp_scrape_port_named_metrics\"\n\t\t}\n\n\t\t// + only keep targets that have scrape: true or \"metrics\" in the port name if the + argument scrape_port_named_metrics\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__tmp_scrape\",\n\t\t\t\t\"__tmp_scrape_port_named_metrics\",\n\t\t\t\t// + endpoints is the role and most meta labels started with \"endpoints\", however + the port name is an exception and starts with \"endpoint\"\n\t\t\t\t\"__meta_kubernetes_\" + + replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"endpoint\") + + \"_port_name\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(true;.*|(|true);true;(.*metrics.*))$\"\n\t\t}\n\n\t\t// + only keep targets where the pod is running or the pod_phase is empty and is not + an init container. This will only exist for role=\"pod\" or\n\t\t// potentially + role=\"endpoints\", if it is a service the value is empty and thus allowed to + pass, if it is an endpoint but not associated to a\n\t\t// pod but rather a static + IP or hostname, that could be outside of kubernetes allow endpoints to declare + what tenant their metrics should be\n\t\t// written to\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex + \ = \"^(?i)(Running|)$\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__meta_kubernetes_pod_ready\"]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\t\t// + if the container is an init container, drop it\n\t\trule {\n\t\t\taction = + \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_container_init\"]\n\t\t\tregex + \ = \"^(true)$\"\n\t\t}\n\n\t\t// allow resources to declare their metrics + the tenant their metrics should be sent to,\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/tenant: primary\n\t\t//\n\t\t// Note: This does not necessarily + have to be the actual tenantId, it can be a friendly name as well that is simply + used\n\t\t// to determine if the metrics should be gathered for the current + tenant\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_tenant\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_tenant\",\n\t\t\t]\n\t\t\tregex + = \"^(\" + coalesce(argument.tenant.value, \".*\") + \")$\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Scrape Metadata i.e. path, port, interval etc.\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare the protocol to use when collecting metrics, the default + value is \"http\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/scheme: + http\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"http\"\n\t\t\ttarget_label + = \"__scheme__\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scheme\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scheme\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?(https?).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scheme__\"\n\t\t}\n\n\t\t// allow resources + to declare the port to use when collecting metrics, the default value is the discovered + port from\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/port: 9090\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__address__\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_port\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_port\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^:]+)(?::\\\\d+)?;(\\\\d+)$\"\n\t\t\treplacement + \ = \"$1:$2\"\n\t\t\ttarget_label = \"__address__\"\n\t\t}\n\n\t\t// allow resources + to declare their the path to use when collecting their metrics, the default value + is \"/metrics\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/path: + /metrics/foo\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_path\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_path\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__metrics_path__\"\n\t\t}\n\n\t\t// allow resources + to declare how often their metrics should be collected, the default value is 1m,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/interval: 5m\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_interval\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_interval\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\t// allow + resources to declare the timeout of the scrape request, the default value is 10s,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/timeout: 30s\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_timeout\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_timeout\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Common Labels\n\t\t****************************************************************************************************************/\n\t\t// + set a source label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = \"kubernetes\"\n\t\t\ttarget_label = \"source\"\n\t\t}\n\n\t\t// set the cluster + label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = argument.cluster.value\n\t\t\ttarget_label + = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label + \ = \"namespace\"\n\t\t}\n\n\t\t// set the target name label i.e. service name, + pod name, etc.\n\t\t// if the role is endpoints, the first valued field is used + which would be __meta_kubernetes_pod_name, if the pod name is empty\n\t\t// then + the endpoint name would be used\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + argument.__pod_role.value + \"_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = + \"$1\"\n\t\t\ttarget_label = argument.__pod_role.value\n\t\t}\n\n\t\t// set a + default job label to be the namespace/pod_controller_name or namespace/service_name\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t\targument.__pod_role.value,\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^;]+)(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// if the controller + is a ReplicaSet, drop the hash from the end of the ReplicaSet\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_type\",\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:ReplicaSet);([^;]+);([^;]+)-.+$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// allow resources to + declare their the job label value to use when collecting their metrics, the default + value is \"\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/job: + integrations/kubernetes/cadvisor\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_job\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_job\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// set the app name if + specified as metadata labels \"app:\" or \"app.kubernetes.io/name:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"app\"\n\t\t}\n\n\t\t// + set the app component if specified as metadata labels \"component:\" or \"app.kubernetes.io/component:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_component\",\n\t\t\t]\n\t\t\tregex + \ = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"component\"\n\t\t}\n\n\t\t// set the version if specified as metadata labels + \"version:\" or \"app.kubernetes.io/version:\" or \"app_version:\"\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_version\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"version\"\n\t\t}\n\n\t\t// + set a workload label if the resource is a pod\n\t\t// example: grafana-agent-68nv9 + becomes DaemonSet/grafana-agent\n\t\trule {\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_kind\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"(.+);(.+)\"\n\t\t\treplacement = \"$1/$2\"\n\t\t\ttarget_label + = \"workload\"\n\t\t}\n\t\t// remove the hash from the ReplicaSet\n\t\trule {\n\t\t\tsource_labels + = [\"workload\"]\n\t\t\tregex = \"(ReplicaSet/.+)-.+\"\n\t\t\ttarget_label + \ = \"workload\"\n\t\t}\n\t}\n\n\t// only keep http targets\n\tdiscovery.relabel + \"http_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"http\"\n\t\t}\n\t}\n\n\t// only keep https targets\n\tdiscovery.relabel + \"https_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"https\"\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Scrape Labels Targets\n\t*****************************************************************/\n\t// + scrape http only targtetsa\n\tprometheus.scrape \"http_annotations\" {\n\t\ttargets + = discovery.relabel.http_annotations.output\n\n\t\tjob_name = \"annotation-metrics-http\"\n\t\tscheme + \ = \"http\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t// + scrape https only targtets\n\tprometheus.scrape \"https_annotations\" {\n\t\ttargets + = discovery.relabel.https_annotations.output\n\n\t\tjob_name = \"annotation-metrics-https\"\n\t\tscheme + \ = \"https\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\tbearer_token_file + = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n\n\t\ttls_config {\n\t\t\tca_file + \ = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n\t\t\tinsecure_skip_verify + = false\n\t\t\tserver_name = \"kubernetes\"\n\t\t}\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"annotations\" {\n\t\tforward_to = argument.forward_to.value\n\t\t// keep only + metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(.+)\")\n\t\t}\n\n\t\t// drop metrics that match the drop_metrics regex\n\t\trule + {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex + \ = coalesce(argument.drop_metrics.value, \"\")\n\t\t}\n\t}\n}\n" podmonitors-scrape.alloy: "/*\nModule Components: podmonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.podmonitors\n*/\n\ndeclare \"podmonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape PodMonitors\n\t*****************************************************************/\n\tprometheus.operator.podmonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.podmonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\t// selector {\n\t\t// \tmatch_expression {\n\t\t// \t\tkey = \"team\"\n\t\t// \t\toperator = \"In\"\n\t\t// \t\tvalues - \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n}\n" + \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"podmonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set the + cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" + rules-to-mimir.alloy: "/*\nModule Components: rules_to_mimir\nDescription: Auto + discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance.\n*/\n\ndeclare + \"rules_to_mimir\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + anonymous)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Mimir\n\t********************************************/\n\tmimir.rules.kubernetes + \"rules_to_mimir\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:8080\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\t// rule_selector {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= + \"true\",\n\t\t// \t}\n\t\t// }\n\t}\n}\n" servicemonitors-scrape.alloy: "/*\nModule Components: servicemonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.servicemonitors\n*/\n\ndeclare \"servicemonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape ServiceMonitors\n\t*****************************************************************/\n\tprometheus.operator.servicemonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.servicemonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering - {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n" + {\n\t\t\tenabled = true\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"servicemonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set + the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f namespace: monitoring-system --- apiVersion: v1 @@ -687,19 +1026,21 @@ data: \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(ProfilessReceiver) where collected - logs should be forwarded to\"\n\t}\n\n\tdiscovery.kubernetes \"pyroscope_kubernetes\" - {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape config allows to define - annotations based scraping.\n\t//\n\t// For example the following annotations:\n\t//\n\t// - ```\n\t// profiles.grafana.com/memory.scrape: \"true\"\n\t// profiles.grafana.com/memory.port: - \"8080\"\n\t// profiles.grafana.com/cpu.scrape: \"true\"\n\t// profiles.grafana.com/cpu.port: - \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: \"true\"\n\t// profiles.grafana.com/goroutine.port: - \"8080\"\n\t// ```\n\t//\n\t// will scrape the `memory`, `cpu` and `goroutine` - profiles from the `8080` port of the pod.\n\t//\n\t// For more information see - https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel + logs should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\tdiscovery.kubernetes + \"pyroscope_kubernetes\" {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape + config allows to define annotations based scraping.\n\t//\n\t// For example the + following annotations:\n\t//\n\t// ```\n\t// profiles.grafana.com/memory.scrape: + \"true\"\n\t// profiles.grafana.com/memory.port: \"8080\"\n\t// profiles.grafana.com/cpu.scrape: + \"true\"\n\t// profiles.grafana.com/cpu.port: \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: + \"true\"\n\t// profiles.grafana.com/goroutine.port: \"8080\"\n\t// ```\n\t//\n\t// + will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port + of the pod.\n\t//\n\t// For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel \"kubernetes_pods\" {\n\t\ttargets = concat(discovery.kubernetes.pyroscope_kubernetes.targets)\n\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex \ = \"Pending|Succeeded|Failed|Completed\"\n\t\t}\n\n\t\trule {\n\t\t\taction - = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\trule + = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\trule @@ -918,7 +1259,7 @@ data: {\n\t\t\t\tenabled = true\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 namespace: monitoring-system --- apiVersion: v1 @@ -928,40 +1269,58 @@ data: Transformation\ndeclare \"process_and_transform\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"traces_forward_to\" {\n\t\tcomment = \"Must be a list(TracesReceiver) where - collected traces should be forwarded to\"\n\t}\n\n\targument \"logs_forward_to\" - {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be - forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment = \"Must - be a list(MetricsReceiver) where collected metrics should be forwarded to\"\n\t}\n\n\targument - \"cluster\" {\n\t\toptional = true\n\t\tdefault = \"k3d-k3s-codelab\"\n\t}\n\n\targument - \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4318\"\n\t}\n\n\targument - \"otlp_grpc_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* + collected traces should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\targument + \"logs_forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected + logs should be forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment + = \"Must be a list(MetricsReceiver) where collected metrics should be forwarded + to\"\n\t}\n\n\targument \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault + \ = \"0.0.0.0:4318\"\n\t}\n\n\targument \"otlp_grpc_endpoint\" {\n\t\toptional + = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* Jaeger for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.jaeger \"default\" {\n\t\tprotocols {\n\t\t\tgrpc {\n\t\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t\t}\n\n\t\t\tthrift_http {\n\t\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t\t}\n\n\t\t\tthrift_binary {\n\t\t\t\tendpoint = \"0.0.0.0:6832\"\n\t\t\t}\n\n\t\t\tthrift_compact {\n\t\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [otelcol.processor.resourcedetection.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + {\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* Otelcol for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.otlp - \"default\" {\n\t\tgrpc {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp - {\n\t\t\tendpoint = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics - = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection - \"default\" {\n\t\tdetectors = [\"env\"]\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces - = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes + \"default\" {\n\t\tdebug_metrics {\n\t\t\tdisable_high_cardinality_metrics = true\n\t\t}\n\n\t\tgrpc + {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp {\n\t\t\tendpoint + = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.resourcedetection.default.input]\n\t\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection + \"default\" {\n\t\tdetectors = [\"env\", \"system\"]\n\n\t\tsystem {\n\t\t\thostname_sources + = [\"os\"]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\n\t\t\tlogs + \ = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform + \"add_metric_datapoint_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements + {\n\t\t\tcontext = \"datapoint\"\n\t\t\tstatements = [\n\t\t\t\t\"set(attributes[\\\"deployment.environment\\\"], + resource.attributes[\\\"deployment.environment\\\"])\",\n\t\t\t\t\"set(attributes[\\\"service.version\\\"], + resource.attributes[\\\"service.version\\\"])\",\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes \"default\" {\n\t\textract {\n\t\t\tmetadata = [\n\t\t\t\t\"k8s.namespace.name\",\n\t\t\t\t\"k8s.pod.name\",\n\t\t\t\t\"k8s.deployment.name\",\n\t\t\t\t\"k8s.statefulset.name\",\n\t\t\t\t\"k8s.daemonset.name\",\n\t\t\t\t\"k8s.cronjob.name\",\n\t\t\t\t\"k8s.job.name\",\n\t\t\t\t\"k8s.node.name\",\n\t\t\t\t\"k8s.pod.uid\",\n\t\t\t\t\"k8s.pod.start_time\",\n\t\t\t]\n\t\t}\n\n\t\tpod_association {\n\t\t\tsource {\n\t\t\t\tfrom = \"connection\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t\ttraces - = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform - \"add_resource_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tlog_statements + {\n\t\t\tmetrics = [otelcol.processor.transform.default.input]\n\t\t\tlogs = + [otelcol.processor.transform.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.transform.default.input,\n\t\t\t\totelcol.connector.host_info.default.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.connector.host_info + \"default\" {\n\t\thost_identifiers = [\"k8s.node.name\"]\n\n\t\toutput {\n\t\t\tmetrics + = [otelcol.processor.batch.host_info_batch.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + \"host_info_batch\" {\n\t\toutput {\n\t\t\tmetrics = [otelcol.exporter.prometheus.host_info_metrics.input]\n\t\t}\n\t}\n\n\totelcol.exporter.prometheus + \"host_info_metrics\" {\n\t\tadd_metric_suffixes = false\n\t\tforward_to = + argument.metrics_forward_to.value\n\t}\n\n\totelcol.processor.transform \"default\" + {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements {\n\t\t\tcontext = + \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\tlog_statements {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])`,\n\t\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t\t`set(attributes[\"loki.resource.labels\"], \"pod, namespace, cluster, job\")`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements - {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], - \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter - \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.batch.default.input]\n\t\t\ttraces - = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements + {\n\t\t\tcontext = \"span\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces + \ = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter + \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\ttraces {\n\t\t\tspan = [\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/live\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] == \\\"/healthy\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/ready\\\"\",\n\t\t\t]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs + \ = [otelcol.processor.batch.default.input]\n\t\t\ttraces = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch \"default\" {\n\t\tsend_batch_size = 16384\n\t\tsend_batch_max_size = 0\n\t\ttimeout \ = \"5s\"\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\t\tlogs \ = [otelcol.processor.memory_limiter.default.input]\n\t\t\ttraces = [otelcol.processor.memory_limiter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.memory_limiter @@ -997,7 +1356,7 @@ data: \"alloy_traces_input\" {\n\t\tvalue = otelcol.processor.batch.default.input\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 namespace: monitoring-system --- apiVersion: v1 @@ -1722,19 +2081,19 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-4m6599mmm2 + name: alloy-config-fmt85fgk68 name: config - configMap: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f name: modules-kubernetes-metrics - configMap: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k name: modules-kubernetes-logs - configMap: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 name: modules-kubernetes-traces - configMap: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 name: modules-kubernetes-profiles --- apiVersion: monitoring.coreos.com/v1 diff --git a/kubernetes/monolithic-mode/logs/logs.alloy b/kubernetes/monolithic-mode/logs/logs.alloy index 178fa156..f7d59991 100644 --- a/kubernetes/monolithic-mode/logs/logs.alloy +++ b/kubernetes/monolithic-mode/logs/logs.alloy @@ -26,13 +26,15 @@ import.file "logs" { } logs.kubernetes_cluster_events "kubernetes" { - cluster = "k3d-k3s-codelab" + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + forward_to = [logs.keep_labels.kubernetes.receiver] } logs.annotations_scrape "kubernetes" { - annotation_prefix = "logs.grafana.com" - forward_to = [logs.keep_labels.kubernetes.receiver] + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + + forward_to = [logs.keep_labels.kubernetes.receiver] } logs.keep_labels "kubernetes" { @@ -46,11 +48,17 @@ import.file "metrics" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/metrics" } -metrics.podmonitors_scrape "kubernetes" { +metrics.annotations_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } metrics.servicemonitors_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } @@ -68,32 +76,8 @@ import.string "memcached" { } memcached.memcached_metrics_scrape "instance" { - forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] - namespace = "monitoring-system" name = remote.kubernetes.configmap.integrations.data["MEMCACHED_K8S_SECRET_NAME"] -} -// // Redis Integrations -// import.string "redis" { -// content = remote.kubernetes.configmap.integrations.data["redis.alloy"] -// } - -// redis.redis_exporter_metrics_scrape "instance" { -// forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] - -// namespace = "monitoring-system" -// name = remote.kubernetes.configmap.integrations.data["REDIS_K8S_SECRET_NAME"] -// } - -// // Mysql Integrations -// import.string "mysql" { -// content = remote.kubernetes.configmap.integrations.data["mysql.alloy"] -// } - -// mysql.mysql_metrics_scrape "instance" { -// forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] - -// namespace = "monitoring-system" -// name = remote.kubernetes.configmap.integrations.data["MYSQL_K8S_SECRET_NAME"] -// } + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] +} diff --git a/kubernetes/monolithic-mode/metrics/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/metrics/k8s-all-in-one.yaml index 519506e2..a8a64750 100644 --- a/kubernetes/monolithic-mode/metrics/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/metrics/k8s-all-in-one.yaml @@ -154,43 +154,38 @@ data: = coalesce(env(\"SELF_HOSTED_METRICS_ENDPOINT_URL\"), \"http://nginx.gateway.svc:8080/api/v1/push\")\n}\n\n/********************************************\n * Metrics\n ********************************************/\nimport.file \"metrics\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/metrics\"\n}\n\nmetrics.podmonitors_scrape \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape - \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\n// - metrics.annotations_scrape \"kubernetes\" {\n// \tlabel_prefix = \"metrics.grafana.com\"\n// - \tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n// - \tscrape_interval = \"15s\"\n// }\n" + + \"/kubernetes/metrics\"\n}\n\nmetrics.annotations_scrape \"kubernetes\" {\n\tforward_to + = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\n\tcluster = + coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval = \"15s\"\n}\n\nmetrics.servicemonitors_scrape + \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-55dd47md5f + name: alloy-config-dkdkkgdd64 namespace: monitoring-system --- apiVersion: v1 data: annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: Scrapes targets for logs based on kubernetes Pod annotations\n\n Annotations:\n - \ logs.grafana.com/ingest: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare + \ logs.grafana.com/scrape: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this - can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\t// arguments - for kubernetes discovery\n\targument \"namespaces\" {\n\t\tcomment = \"The namespaces - to look for targets in (default: [\\\"kube-system\\\"] is all namespaces)\"\n\t\toptional - = true\n\t}\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\targument + \"cluster\" { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix to use (default: logs.grafana.com)\"\n\t\tdefault = \"logs.grafana.com\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional = true\n\t\tcomment \ = \"The logic is used to transform the annotation argument into a valid label name by removing unsupported characters.\"\n\t\tdefault = replace(replace(replace(coalesce(argument.annotation_prefix.value, \"logs.grafana.com\"), \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\t// - find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\n\t\tnamespaces - {\n\t\t\tnames = coalesce(argument.namespaces.value, [])\n\t\t}\n\t}\n\n\t// filter - logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" + find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\t}\n\n\t// + filter logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" {\n\t\ttargets = discovery.kubernetes.annotation_logs.targets\n\n\t\t// allow pods to declare their logs to be ingested or not, the default is true\n\t\t// - \ i.e. logs.grafana.com/ingest: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + \ i.e. logs.grafana.com/scrape: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\n\t\t// allow pods to declare what tenant their logs should be written to, the following annotation @@ -200,6 +195,8 @@ data: + argument.tenant.value + \")$\"\n\t\t}\n\n\t\t// set the instance label as the name of the worker node the pod is on\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_node_name\"]\n\t\t\ttarget_label = \"instance\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\t// set the pod label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\t// @@ -264,12 +261,11 @@ data: ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\toptional - = true\n\t}\n\n\targument \"cluster\" {\n\t\toptional = true\n\t}\n\n\tloki.source.kubernetes_events - \"cluster_events\" {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format + = true\n\t}\n\n\targument \"cluster\" { }\n\n\tloki.source.kubernetes_events \"cluster_events\" + {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format = \"logfmt\"\n\t\tforward_to = [loki.process.logs_service.receiver]\n\t}\n\n\tloki.process \"logs_service\" {\n\t\tstage.static_labels {\n\t\t\tvalues = {\n\t\t\t\tcluster - = coalesce(argument.cluster.value, \"k3d\"),\n\t\t\t}\n\t\t}\n\t\tforward_to = - argument.forward_to.value\n\t}\n}\n" + = argument.cluster.value,\n\t\t\t}\n\t\t}\n\t\tforward_to = argument.forward_to.value\n\t}\n}\n" keep-labels.alloy: "/*\nModule Components: keep_labels\nDescription: Pre-defined set of labels to keep, this stage should always be in-place as the previous relabeing\n \ stages make every pod label and annotation a label in the pipeline, @@ -291,46 +287,392 @@ data: = argument.keep_labels.value\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* EXPORTS\n\t*****************************************************************/\n\texport \"receiver\" {\n\t\tvalue = loki.process.keep_labels.receiver\n\t}\n}\n" + rules-to-loki.alloy: "/*\nModule Components: rules_to_loki\nDescription: Auto discovers + PrometheusRule Kubernetes resources and loads them into a Loki instance.\n*/\n\ndeclare + \"rules_to_loki\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + fake)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Loki\n\t********************************************/\n\tloki.rules.kubernetes + \"rules_to_loki\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:3100\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_loki= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\trule_selector {\n\t\t\tmatch_labels = {\n\t\t\t\tauto_rules_to_loki + = \"true\",\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k namespace: monitoring-system --- apiVersion: v1 data: + annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: + Scrapes targets for metrics based on kubernetes annotations\n\nNote: Every argument + except for \"forward_to\" is optional, and does have a defined default value. + \ However, the values for these\n arguments are not defined using the default + = \" ... \" argument syntax, but rather using the coalesce(argument.value, \" + ... \").\n This is because if the argument passed in from another consuming + module is set to null, the default = \" ... \" syntax will\n does not override + the value passed in, where coalesce() will return the first non-null value.\n\nKubernetes + Annotation Auto-Scraping\n------------------------------------------------------------------------------------------------------------------------------------\nThis + module is meant to be used to automatically scrape targets based on a certain + role and set of annotations. This module can be consumed\nmultiple times with + different roles. The supported roles are:\n\n - pod\n - service\n - endpoints\n\nTypically, + if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality + you would use role=\"endpoints\", if\nmimicking the behavior of the PodMonitor + functionality you would use role=\"pod\". It is important to understand that + with endpoints,\nthe target is typically going to be a pod, and whatever annotations + that are set on the service will automatically be propagated to the\nendpoints. + \ This is why the role \"endpoints\" is used, because it will scrape the pod, + but also consider the service annotations. Using\nrole=\"endpoints\", which scrape + each endpoint associated to the service. If role=\"service\" is used, it will + only scrape the service, only\nhitting one of the endpoints associated to the + service.\n\nThis is where you must consider your scraping strategy, for example + if you scrape a service like \"kube-state-metrics\" using\nrole=\"endpoints\" + you should only have a single replica of the kube-state-metrics pod, if you have + multiple replicas, you should use\nrole=\"service\" or a separate non-annotation + job completely. Scraping a service instead of endpoints, is typically a rare + use case, but\nit is supported.\n\nThere are other considerations for using annotation + based scraping as well, which is metric relabeling rules that happen post scrape. + \ If\nyou have a target that you want to apply a bunch of relabelings to or a + very large metrics response payload, performance wise it will be\nbetter to have + a separate job for that target, rather than using use annotations. As every targert + will go through the ssame relabeling.\nTypical deployment strategies/options would + be:\n\nOption #1 (recommended):\n - Annotation Scraping for role=\"endpoints\"\n + \ - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, + etc.) or large metric payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, + kube-apiserver, kube-scheduler, etc.)\n\nOption #2:\n - Annotation Scraping for + role=\"pod\"\n - Annotation Scraping for role=\"service\" (i.e. kube-state-metrics, + node-exporter, etc.)\n - Separate Jobs for specific use cases or large metric + payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, + kube-scheduler, etc.)\n\nAt no point should you use role=\"endpoints\" and role=\"pod\" + together, as this will result in duplicate targets being scraped, thus\ngenerating + duplicate metrics. If you want to scrape both the pod and the service, use Option + #2.\n\nEach port attached to an service/pod/endpoint is an eligible target, oftentimes + it will have multiple ports.\nThere may be instances when you want to scrape all + ports or some ports and not others. To support this\nthe following annotations + are available:\n\n metrics.grafana.com/scrape: true\n\nthe default scraping scheme + is http, this can be specified as a single value which would override, the schema + being used for all\nports attached to the target:\n\n metrics.grafana.com/scheme: + https\n\nthe default path to scrape is /metrics, this can be specified as a single + value which would override, the scrape path being used\nfor all ports attached + to the target:\n\n metrics.grafana.com/path: /metrics/some_path\n\nthe default + port to scrape is the target port, this can be specified as a single value which + would override the scrape port being\nused for all ports attached to the target, + note that even if aan target had multiple targets, the relabel_config targets + are\ndeduped before scraping:\n\n metrics.grafana.com/port: 8080\n\nthe default + interval to scrape is 1m, this can be specified as a single value which would + override, the scrape interval being used\nfor all ports attached to the target:\n\n + \ metrics.grafana.com/interval: 5m\n\nthe default timeout for scraping is 10s, + this can be specified as a single value which would override, the scrape interval + being\nused for all ports attached to the target:\n\n metrics.grafana.com/timeout: + 30s\n\nthe default job is namespace/{{ service name }} or namespace/{{ controller_name + }} depending on the role, there may be instances\nin which a different job name + is required because of a set of dashboards, rules, etc. to support this there + is a job annotation\nwhich will override the default value:\n\n metrics.grafana.com/job: + integrations/kubernetes/kube-state-metrics\n*/\n\ndeclare \"annotations_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment + \ = \"The tenant to filter metrics to. This does not have to be the tenantId, + this is the value to look for in the metrics.agent.grafana.com/tenant annotation, + and this can be a regex.\"\n\t\toptional = true\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + to use (default: metrics.grafana.com)\"\n\t\tdefault = \"metrics.grafana.com\"\n\t\toptional + = true\n\t}\n\n\targument \"role\" {\n\t\tcomment = \"The role to use when looking + for targets to scrape via annotations, can be: endpoints, service, pod (default: + endpoints)\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional + = true\n\t\tcomment = \"The logic is used to transform the annotation argument + into a valid label name by removing unsupported characters.\"\n\t\tdefault = + replace(replace(replace(coalesce(argument.annotation_prefix.value, \"metrics.grafana.com\"), + \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\targument \"__pod_role\" + {\n\t\tcomment = \"Most annotation targets service or pod that is all you want, + however if the role is endpoints you want the pod\"\n\t\toptional = true\n\t\tdefault + \ = replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"pod\")\n\t}\n\n\targument + \"__service_role\" {\n\t\tcomment = \"Most annotation targets service or pod + that is all you want, however if the role is endpoints you we also want to consider + service annotations\"\n\t\toptional = true\n\t\tdefault = replace(coalesce(argument.role.value, + \"endpoints\"), \"endpoints\", \"service\")\n\t}\n\n\targument \"scrape_port_named_metrics\" + {\n\t\tcomment = \"Whether or not to automatically scrape endpoints that have + a port with 'metrics' in the name\"\n\t\toptional = true\n\t\tdefault = false\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: (.+))\"\n\t\toptional + = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment = \"A regex of metrics + to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" + {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + Targets From Docker Discovery\n\t*****************************************************************/\n\tdiscovery.kubernetes + \"annotation_metrics\" {\n\t\trole = coalesce(argument.role.value, \"endpoints\")\n\n\t\tselectors + {\n\t\t\trole = coalesce(argument.role.value, \"endpoints\")\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t*****************************************************************/\n\t// + filter metrics by kubernetes annotations\n\tdiscovery.relabel \"annotation_metrics_filter\" + {\n\t\ttargets = discovery.kubernetes.annotation_metrics.targets\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Targets to Keep or Drop\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare their metrics scraped or not\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/scrape: false\n\t\t//\n\t\t// the label prometheus.io/service-monitor: + \"false\" is a common label for headless services, when performing endpoint\n\t\t// + service discovery, if there is both a load-balanced service and headless service, + this can result in duplicate\n\t\t// scrapes if the name of the service is attached + as a label. any targets with this label or annotation set should be dropped\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"false\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, + \"endpoints\") + \"_label_prometheus_io_service_monitor\",\n\t\t\t]\n\t\t\tseparator + = \";\"\n\t\t\t// only allow empty or true, otherwise defaults to false\n\t\t\tregex + \ = \"^(?:;*)?(true)(;|true)*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\t// add a __tmp_scrape_port_named_metrics from + the argument.scrape_port_named_metrics\n\t\trule {\n\t\t\treplacement = format(\"%t\", + argument.scrape_port_named_metrics.value)\n\t\t\ttarget_label = \"__tmp_scrape_port_named_metrics\"\n\t\t}\n\n\t\t// + only keep targets that have scrape: true or \"metrics\" in the port name if the + argument scrape_port_named_metrics\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__tmp_scrape\",\n\t\t\t\t\"__tmp_scrape_port_named_metrics\",\n\t\t\t\t// + endpoints is the role and most meta labels started with \"endpoints\", however + the port name is an exception and starts with \"endpoint\"\n\t\t\t\t\"__meta_kubernetes_\" + + replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"endpoint\") + + \"_port_name\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(true;.*|(|true);true;(.*metrics.*))$\"\n\t\t}\n\n\t\t// + only keep targets where the pod is running or the pod_phase is empty and is not + an init container. This will only exist for role=\"pod\" or\n\t\t// potentially + role=\"endpoints\", if it is a service the value is empty and thus allowed to + pass, if it is an endpoint but not associated to a\n\t\t// pod but rather a static + IP or hostname, that could be outside of kubernetes allow endpoints to declare + what tenant their metrics should be\n\t\t// written to\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex + \ = \"^(?i)(Running|)$\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__meta_kubernetes_pod_ready\"]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\t\t// + if the container is an init container, drop it\n\t\trule {\n\t\t\taction = + \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_container_init\"]\n\t\t\tregex + \ = \"^(true)$\"\n\t\t}\n\n\t\t// allow resources to declare their metrics + the tenant their metrics should be sent to,\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/tenant: primary\n\t\t//\n\t\t// Note: This does not necessarily + have to be the actual tenantId, it can be a friendly name as well that is simply + used\n\t\t// to determine if the metrics should be gathered for the current + tenant\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_tenant\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_tenant\",\n\t\t\t]\n\t\t\tregex + = \"^(\" + coalesce(argument.tenant.value, \".*\") + \")$\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Scrape Metadata i.e. path, port, interval etc.\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare the protocol to use when collecting metrics, the default + value is \"http\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/scheme: + http\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"http\"\n\t\t\ttarget_label + = \"__scheme__\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scheme\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scheme\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?(https?).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scheme__\"\n\t\t}\n\n\t\t// allow resources + to declare the port to use when collecting metrics, the default value is the discovered + port from\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/port: 9090\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__address__\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_port\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_port\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^:]+)(?::\\\\d+)?;(\\\\d+)$\"\n\t\t\treplacement + \ = \"$1:$2\"\n\t\t\ttarget_label = \"__address__\"\n\t\t}\n\n\t\t// allow resources + to declare their the path to use when collecting their metrics, the default value + is \"/metrics\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/path: + /metrics/foo\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_path\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_path\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__metrics_path__\"\n\t\t}\n\n\t\t// allow resources + to declare how often their metrics should be collected, the default value is 1m,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/interval: 5m\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_interval\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_interval\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\t// allow + resources to declare the timeout of the scrape request, the default value is 10s,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/timeout: 30s\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_timeout\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_timeout\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Common Labels\n\t\t****************************************************************************************************************/\n\t\t// + set a source label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = \"kubernetes\"\n\t\t\ttarget_label = \"source\"\n\t\t}\n\n\t\t// set the cluster + label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = argument.cluster.value\n\t\t\ttarget_label + = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label + \ = \"namespace\"\n\t\t}\n\n\t\t// set the target name label i.e. service name, + pod name, etc.\n\t\t// if the role is endpoints, the first valued field is used + which would be __meta_kubernetes_pod_name, if the pod name is empty\n\t\t// then + the endpoint name would be used\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + argument.__pod_role.value + \"_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = + \"$1\"\n\t\t\ttarget_label = argument.__pod_role.value\n\t\t}\n\n\t\t// set a + default job label to be the namespace/pod_controller_name or namespace/service_name\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t\targument.__pod_role.value,\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^;]+)(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// if the controller + is a ReplicaSet, drop the hash from the end of the ReplicaSet\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_type\",\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:ReplicaSet);([^;]+);([^;]+)-.+$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// allow resources to + declare their the job label value to use when collecting their metrics, the default + value is \"\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/job: + integrations/kubernetes/cadvisor\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_job\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_job\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// set the app name if + specified as metadata labels \"app:\" or \"app.kubernetes.io/name:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"app\"\n\t\t}\n\n\t\t// + set the app component if specified as metadata labels \"component:\" or \"app.kubernetes.io/component:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_component\",\n\t\t\t]\n\t\t\tregex + \ = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"component\"\n\t\t}\n\n\t\t// set the version if specified as metadata labels + \"version:\" or \"app.kubernetes.io/version:\" or \"app_version:\"\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_version\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"version\"\n\t\t}\n\n\t\t// + set a workload label if the resource is a pod\n\t\t// example: grafana-agent-68nv9 + becomes DaemonSet/grafana-agent\n\t\trule {\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_kind\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"(.+);(.+)\"\n\t\t\treplacement = \"$1/$2\"\n\t\t\ttarget_label + = \"workload\"\n\t\t}\n\t\t// remove the hash from the ReplicaSet\n\t\trule {\n\t\t\tsource_labels + = [\"workload\"]\n\t\t\tregex = \"(ReplicaSet/.+)-.+\"\n\t\t\ttarget_label + \ = \"workload\"\n\t\t}\n\t}\n\n\t// only keep http targets\n\tdiscovery.relabel + \"http_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"http\"\n\t\t}\n\t}\n\n\t// only keep https targets\n\tdiscovery.relabel + \"https_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"https\"\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Scrape Labels Targets\n\t*****************************************************************/\n\t// + scrape http only targtetsa\n\tprometheus.scrape \"http_annotations\" {\n\t\ttargets + = discovery.relabel.http_annotations.output\n\n\t\tjob_name = \"annotation-metrics-http\"\n\t\tscheme + \ = \"http\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t// + scrape https only targtets\n\tprometheus.scrape \"https_annotations\" {\n\t\ttargets + = discovery.relabel.https_annotations.output\n\n\t\tjob_name = \"annotation-metrics-https\"\n\t\tscheme + \ = \"https\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\tbearer_token_file + = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n\n\t\ttls_config {\n\t\t\tca_file + \ = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n\t\t\tinsecure_skip_verify + = false\n\t\t\tserver_name = \"kubernetes\"\n\t\t}\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"annotations\" {\n\t\tforward_to = argument.forward_to.value\n\t\t// keep only + metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(.+)\")\n\t\t}\n\n\t\t// drop metrics that match the drop_metrics regex\n\t\trule + {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex + \ = coalesce(argument.drop_metrics.value, \"\")\n\t\t}\n\t}\n}\n" podmonitors-scrape.alloy: "/*\nModule Components: podmonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.podmonitors\n*/\n\ndeclare \"podmonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape PodMonitors\n\t*****************************************************************/\n\tprometheus.operator.podmonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.podmonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\t// selector {\n\t\t// \tmatch_expression {\n\t\t// \t\tkey = \"team\"\n\t\t// \t\toperator = \"In\"\n\t\t// \t\tvalues - \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n}\n" + \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"podmonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set the + cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" + rules-to-mimir.alloy: "/*\nModule Components: rules_to_mimir\nDescription: Auto + discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance.\n*/\n\ndeclare + \"rules_to_mimir\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + anonymous)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Mimir\n\t********************************************/\n\tmimir.rules.kubernetes + \"rules_to_mimir\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:8080\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\t// rule_selector {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= + \"true\",\n\t\t// \t}\n\t\t// }\n\t}\n}\n" servicemonitors-scrape.alloy: "/*\nModule Components: servicemonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.servicemonitors\n*/\n\ndeclare \"servicemonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape ServiceMonitors\n\t*****************************************************************/\n\tprometheus.operator.servicemonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.servicemonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering - {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n" + {\n\t\t\tenabled = true\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"servicemonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set + the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f namespace: monitoring-system --- apiVersion: v1 @@ -340,19 +682,21 @@ data: \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(ProfilessReceiver) where collected - logs should be forwarded to\"\n\t}\n\n\tdiscovery.kubernetes \"pyroscope_kubernetes\" - {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape config allows to define - annotations based scraping.\n\t//\n\t// For example the following annotations:\n\t//\n\t// - ```\n\t// profiles.grafana.com/memory.scrape: \"true\"\n\t// profiles.grafana.com/memory.port: - \"8080\"\n\t// profiles.grafana.com/cpu.scrape: \"true\"\n\t// profiles.grafana.com/cpu.port: - \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: \"true\"\n\t// profiles.grafana.com/goroutine.port: - \"8080\"\n\t// ```\n\t//\n\t// will scrape the `memory`, `cpu` and `goroutine` - profiles from the `8080` port of the pod.\n\t//\n\t// For more information see - https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel + logs should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\tdiscovery.kubernetes + \"pyroscope_kubernetes\" {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape + config allows to define annotations based scraping.\n\t//\n\t// For example the + following annotations:\n\t//\n\t// ```\n\t// profiles.grafana.com/memory.scrape: + \"true\"\n\t// profiles.grafana.com/memory.port: \"8080\"\n\t// profiles.grafana.com/cpu.scrape: + \"true\"\n\t// profiles.grafana.com/cpu.port: \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: + \"true\"\n\t// profiles.grafana.com/goroutine.port: \"8080\"\n\t// ```\n\t//\n\t// + will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port + of the pod.\n\t//\n\t// For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel \"kubernetes_pods\" {\n\t\ttargets = concat(discovery.kubernetes.pyroscope_kubernetes.targets)\n\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex \ = \"Pending|Succeeded|Failed|Completed\"\n\t\t}\n\n\t\trule {\n\t\t\taction - = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\trule + = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\trule @@ -571,7 +915,7 @@ data: {\n\t\t\t\tenabled = true\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 namespace: monitoring-system --- apiVersion: v1 @@ -581,40 +925,58 @@ data: Transformation\ndeclare \"process_and_transform\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"traces_forward_to\" {\n\t\tcomment = \"Must be a list(TracesReceiver) where - collected traces should be forwarded to\"\n\t}\n\n\targument \"logs_forward_to\" - {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be - forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment = \"Must - be a list(MetricsReceiver) where collected metrics should be forwarded to\"\n\t}\n\n\targument - \"cluster\" {\n\t\toptional = true\n\t\tdefault = \"k3d-k3s-codelab\"\n\t}\n\n\targument - \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4318\"\n\t}\n\n\targument - \"otlp_grpc_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* + collected traces should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\targument + \"logs_forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected + logs should be forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment + = \"Must be a list(MetricsReceiver) where collected metrics should be forwarded + to\"\n\t}\n\n\targument \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault + \ = \"0.0.0.0:4318\"\n\t}\n\n\targument \"otlp_grpc_endpoint\" {\n\t\toptional + = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* Jaeger for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.jaeger \"default\" {\n\t\tprotocols {\n\t\t\tgrpc {\n\t\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t\t}\n\n\t\t\tthrift_http {\n\t\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t\t}\n\n\t\t\tthrift_binary {\n\t\t\t\tendpoint = \"0.0.0.0:6832\"\n\t\t\t}\n\n\t\t\tthrift_compact {\n\t\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [otelcol.processor.resourcedetection.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + {\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* Otelcol for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.otlp - \"default\" {\n\t\tgrpc {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp - {\n\t\t\tendpoint = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics - = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection - \"default\" {\n\t\tdetectors = [\"env\"]\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces - = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes + \"default\" {\n\t\tdebug_metrics {\n\t\t\tdisable_high_cardinality_metrics = true\n\t\t}\n\n\t\tgrpc + {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp {\n\t\t\tendpoint + = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.resourcedetection.default.input]\n\t\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection + \"default\" {\n\t\tdetectors = [\"env\", \"system\"]\n\n\t\tsystem {\n\t\t\thostname_sources + = [\"os\"]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\n\t\t\tlogs + \ = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform + \"add_metric_datapoint_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements + {\n\t\t\tcontext = \"datapoint\"\n\t\t\tstatements = [\n\t\t\t\t\"set(attributes[\\\"deployment.environment\\\"], + resource.attributes[\\\"deployment.environment\\\"])\",\n\t\t\t\t\"set(attributes[\\\"service.version\\\"], + resource.attributes[\\\"service.version\\\"])\",\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes \"default\" {\n\t\textract {\n\t\t\tmetadata = [\n\t\t\t\t\"k8s.namespace.name\",\n\t\t\t\t\"k8s.pod.name\",\n\t\t\t\t\"k8s.deployment.name\",\n\t\t\t\t\"k8s.statefulset.name\",\n\t\t\t\t\"k8s.daemonset.name\",\n\t\t\t\t\"k8s.cronjob.name\",\n\t\t\t\t\"k8s.job.name\",\n\t\t\t\t\"k8s.node.name\",\n\t\t\t\t\"k8s.pod.uid\",\n\t\t\t\t\"k8s.pod.start_time\",\n\t\t\t]\n\t\t}\n\n\t\tpod_association {\n\t\t\tsource {\n\t\t\t\tfrom = \"connection\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t\ttraces - = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform - \"add_resource_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tlog_statements + {\n\t\t\tmetrics = [otelcol.processor.transform.default.input]\n\t\t\tlogs = + [otelcol.processor.transform.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.transform.default.input,\n\t\t\t\totelcol.connector.host_info.default.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.connector.host_info + \"default\" {\n\t\thost_identifiers = [\"k8s.node.name\"]\n\n\t\toutput {\n\t\t\tmetrics + = [otelcol.processor.batch.host_info_batch.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + \"host_info_batch\" {\n\t\toutput {\n\t\t\tmetrics = [otelcol.exporter.prometheus.host_info_metrics.input]\n\t\t}\n\t}\n\n\totelcol.exporter.prometheus + \"host_info_metrics\" {\n\t\tadd_metric_suffixes = false\n\t\tforward_to = + argument.metrics_forward_to.value\n\t}\n\n\totelcol.processor.transform \"default\" + {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements {\n\t\t\tcontext = + \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\tlog_statements {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])`,\n\t\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t\t`set(attributes[\"loki.resource.labels\"], \"pod, namespace, cluster, job\")`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements - {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], - \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter - \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.batch.default.input]\n\t\t\ttraces - = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements + {\n\t\t\tcontext = \"span\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces + \ = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter + \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\ttraces {\n\t\t\tspan = [\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/live\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] == \\\"/healthy\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/ready\\\"\",\n\t\t\t]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs + \ = [otelcol.processor.batch.default.input]\n\t\t\ttraces = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch \"default\" {\n\t\tsend_batch_size = 16384\n\t\tsend_batch_max_size = 0\n\t\ttimeout \ = \"5s\"\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\t\tlogs \ = [otelcol.processor.memory_limiter.default.input]\n\t\t\ttraces = [otelcol.processor.memory_limiter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.memory_limiter @@ -650,7 +1012,7 @@ data: \"alloy_traces_input\" {\n\t\tvalue = otelcol.processor.batch.default.input\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 namespace: monitoring-system --- apiVersion: v1 @@ -1134,19 +1496,19 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-55dd47md5f + name: alloy-config-dkdkkgdd64 name: config - configMap: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f name: modules-kubernetes-metrics - configMap: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k name: modules-kubernetes-logs - configMap: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 name: modules-kubernetes-traces - configMap: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 name: modules-kubernetes-profiles --- apiVersion: monitoring.coreos.com/v1 diff --git a/kubernetes/monolithic-mode/metrics/metrics.alloy b/kubernetes/monolithic-mode/metrics/metrics.alloy index 606728c9..b37ad416 100644 --- a/kubernetes/monolithic-mode/metrics/metrics.alloy +++ b/kubernetes/monolithic-mode/metrics/metrics.alloy @@ -24,16 +24,13 @@ import.file "metrics" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/metrics" } -metrics.podmonitors_scrape "kubernetes" { +metrics.annotations_scrape "kubernetes" { forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] + + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" } metrics.servicemonitors_scrape "kubernetes" { forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } - -// metrics.annotations_scrape "kubernetes" { -// label_prefix = "metrics.grafana.com" -// forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] -// scrape_interval = "15s" -// } diff --git a/kubernetes/monolithic-mode/profiles/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/profiles/k8s-all-in-one.yaml index 96bb12ac..ea642657 100644 --- a/kubernetes/monolithic-mode/profiles/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/profiles/k8s-all-in-one.yaml @@ -218,44 +218,43 @@ data: = coalesce(env(\"SELF_HOSTED_PROFILES_ENDPOINT_URL\"), \"http://nginx.gateway.svc:4040\")\n}\n\n/********************************************\n * Profiles\n ********************************************/\nimport.file \"profiles\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/profiles\"\n}\n\nprofiles.annotations_scrape \"kubernetes\" {\n\t// - annotation_prefix = \"profiles.grafana.com\"\n\tforward_to = [provider.self_hosted_stack.kubernetes.profiles_receiver]\n}\n\n/********************************************\n + + \"/kubernetes/profiles\"\n}\n\nprofiles.annotations_scrape \"kubernetes\" {\n\tcluster + = \"k3d-k3s-codelab\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.profiles_receiver]\n}\n\n/********************************************\n * Metrics\n ********************************************/\nimport.file \"metrics\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/metrics\"\n}\n\nmetrics.podmonitors_scrape \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape - \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" + + \"/kubernetes/metrics\"\n}\n\nmetrics.annotations_scrape \"kubernetes\" {\n\tcluster + \ = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-56888m5g8t + name: alloy-config-m5kk6k2f7k namespace: monitoring-system --- apiVersion: v1 data: annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: Scrapes targets for logs based on kubernetes Pod annotations\n\n Annotations:\n - \ logs.grafana.com/ingest: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare + \ logs.grafana.com/scrape: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this - can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\t// arguments - for kubernetes discovery\n\targument \"namespaces\" {\n\t\tcomment = \"The namespaces - to look for targets in (default: [\\\"kube-system\\\"] is all namespaces)\"\n\t\toptional - = true\n\t}\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\targument + \"cluster\" { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix to use (default: logs.grafana.com)\"\n\t\tdefault = \"logs.grafana.com\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional = true\n\t\tcomment \ = \"The logic is used to transform the annotation argument into a valid label name by removing unsupported characters.\"\n\t\tdefault = replace(replace(replace(coalesce(argument.annotation_prefix.value, \"logs.grafana.com\"), \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\t// - find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\n\t\tnamespaces - {\n\t\t\tnames = coalesce(argument.namespaces.value, [])\n\t\t}\n\t}\n\n\t// filter - logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" + find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\t}\n\n\t// + filter logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" {\n\t\ttargets = discovery.kubernetes.annotation_logs.targets\n\n\t\t// allow pods to declare their logs to be ingested or not, the default is true\n\t\t// - \ i.e. logs.grafana.com/ingest: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + \ i.e. logs.grafana.com/scrape: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\n\t\t// allow pods to declare what tenant their logs should be written to, the following annotation @@ -265,6 +264,8 @@ data: + argument.tenant.value + \")$\"\n\t\t}\n\n\t\t// set the instance label as the name of the worker node the pod is on\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_node_name\"]\n\t\t\ttarget_label = \"instance\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\t// set the pod label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\t// @@ -329,12 +330,11 @@ data: ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\toptional - = true\n\t}\n\n\targument \"cluster\" {\n\t\toptional = true\n\t}\n\n\tloki.source.kubernetes_events - \"cluster_events\" {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format + = true\n\t}\n\n\targument \"cluster\" { }\n\n\tloki.source.kubernetes_events \"cluster_events\" + {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format = \"logfmt\"\n\t\tforward_to = [loki.process.logs_service.receiver]\n\t}\n\n\tloki.process \"logs_service\" {\n\t\tstage.static_labels {\n\t\t\tvalues = {\n\t\t\t\tcluster - = coalesce(argument.cluster.value, \"k3d\"),\n\t\t\t}\n\t\t}\n\t\tforward_to = - argument.forward_to.value\n\t}\n}\n" + = argument.cluster.value,\n\t\t\t}\n\t\t}\n\t\tforward_to = argument.forward_to.value\n\t}\n}\n" keep-labels.alloy: "/*\nModule Components: keep_labels\nDescription: Pre-defined set of labels to keep, this stage should always be in-place as the previous relabeing\n \ stages make every pod label and annotation a label in the pipeline, @@ -356,46 +356,392 @@ data: = argument.keep_labels.value\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* EXPORTS\n\t*****************************************************************/\n\texport \"receiver\" {\n\t\tvalue = loki.process.keep_labels.receiver\n\t}\n}\n" + rules-to-loki.alloy: "/*\nModule Components: rules_to_loki\nDescription: Auto discovers + PrometheusRule Kubernetes resources and loads them into a Loki instance.\n*/\n\ndeclare + \"rules_to_loki\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + fake)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Loki\n\t********************************************/\n\tloki.rules.kubernetes + \"rules_to_loki\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:3100\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_loki= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\trule_selector {\n\t\t\tmatch_labels = {\n\t\t\t\tauto_rules_to_loki + = \"true\",\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k namespace: monitoring-system --- apiVersion: v1 data: + annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: + Scrapes targets for metrics based on kubernetes annotations\n\nNote: Every argument + except for \"forward_to\" is optional, and does have a defined default value. + \ However, the values for these\n arguments are not defined using the default + = \" ... \" argument syntax, but rather using the coalesce(argument.value, \" + ... \").\n This is because if the argument passed in from another consuming + module is set to null, the default = \" ... \" syntax will\n does not override + the value passed in, where coalesce() will return the first non-null value.\n\nKubernetes + Annotation Auto-Scraping\n------------------------------------------------------------------------------------------------------------------------------------\nThis + module is meant to be used to automatically scrape targets based on a certain + role and set of annotations. This module can be consumed\nmultiple times with + different roles. The supported roles are:\n\n - pod\n - service\n - endpoints\n\nTypically, + if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality + you would use role=\"endpoints\", if\nmimicking the behavior of the PodMonitor + functionality you would use role=\"pod\". It is important to understand that + with endpoints,\nthe target is typically going to be a pod, and whatever annotations + that are set on the service will automatically be propagated to the\nendpoints. + \ This is why the role \"endpoints\" is used, because it will scrape the pod, + but also consider the service annotations. Using\nrole=\"endpoints\", which scrape + each endpoint associated to the service. If role=\"service\" is used, it will + only scrape the service, only\nhitting one of the endpoints associated to the + service.\n\nThis is where you must consider your scraping strategy, for example + if you scrape a service like \"kube-state-metrics\" using\nrole=\"endpoints\" + you should only have a single replica of the kube-state-metrics pod, if you have + multiple replicas, you should use\nrole=\"service\" or a separate non-annotation + job completely. Scraping a service instead of endpoints, is typically a rare + use case, but\nit is supported.\n\nThere are other considerations for using annotation + based scraping as well, which is metric relabeling rules that happen post scrape. + \ If\nyou have a target that you want to apply a bunch of relabelings to or a + very large metrics response payload, performance wise it will be\nbetter to have + a separate job for that target, rather than using use annotations. As every targert + will go through the ssame relabeling.\nTypical deployment strategies/options would + be:\n\nOption #1 (recommended):\n - Annotation Scraping for role=\"endpoints\"\n + \ - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, + etc.) or large metric payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, + kube-apiserver, kube-scheduler, etc.)\n\nOption #2:\n - Annotation Scraping for + role=\"pod\"\n - Annotation Scraping for role=\"service\" (i.e. kube-state-metrics, + node-exporter, etc.)\n - Separate Jobs for specific use cases or large metric + payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, + kube-scheduler, etc.)\n\nAt no point should you use role=\"endpoints\" and role=\"pod\" + together, as this will result in duplicate targets being scraped, thus\ngenerating + duplicate metrics. If you want to scrape both the pod and the service, use Option + #2.\n\nEach port attached to an service/pod/endpoint is an eligible target, oftentimes + it will have multiple ports.\nThere may be instances when you want to scrape all + ports or some ports and not others. To support this\nthe following annotations + are available:\n\n metrics.grafana.com/scrape: true\n\nthe default scraping scheme + is http, this can be specified as a single value which would override, the schema + being used for all\nports attached to the target:\n\n metrics.grafana.com/scheme: + https\n\nthe default path to scrape is /metrics, this can be specified as a single + value which would override, the scrape path being used\nfor all ports attached + to the target:\n\n metrics.grafana.com/path: /metrics/some_path\n\nthe default + port to scrape is the target port, this can be specified as a single value which + would override the scrape port being\nused for all ports attached to the target, + note that even if aan target had multiple targets, the relabel_config targets + are\ndeduped before scraping:\n\n metrics.grafana.com/port: 8080\n\nthe default + interval to scrape is 1m, this can be specified as a single value which would + override, the scrape interval being used\nfor all ports attached to the target:\n\n + \ metrics.grafana.com/interval: 5m\n\nthe default timeout for scraping is 10s, + this can be specified as a single value which would override, the scrape interval + being\nused for all ports attached to the target:\n\n metrics.grafana.com/timeout: + 30s\n\nthe default job is namespace/{{ service name }} or namespace/{{ controller_name + }} depending on the role, there may be instances\nin which a different job name + is required because of a set of dashboards, rules, etc. to support this there + is a job annotation\nwhich will override the default value:\n\n metrics.grafana.com/job: + integrations/kubernetes/kube-state-metrics\n*/\n\ndeclare \"annotations_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment + \ = \"The tenant to filter metrics to. This does not have to be the tenantId, + this is the value to look for in the metrics.agent.grafana.com/tenant annotation, + and this can be a regex.\"\n\t\toptional = true\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + to use (default: metrics.grafana.com)\"\n\t\tdefault = \"metrics.grafana.com\"\n\t\toptional + = true\n\t}\n\n\targument \"role\" {\n\t\tcomment = \"The role to use when looking + for targets to scrape via annotations, can be: endpoints, service, pod (default: + endpoints)\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional + = true\n\t\tcomment = \"The logic is used to transform the annotation argument + into a valid label name by removing unsupported characters.\"\n\t\tdefault = + replace(replace(replace(coalesce(argument.annotation_prefix.value, \"metrics.grafana.com\"), + \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\targument \"__pod_role\" + {\n\t\tcomment = \"Most annotation targets service or pod that is all you want, + however if the role is endpoints you want the pod\"\n\t\toptional = true\n\t\tdefault + \ = replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"pod\")\n\t}\n\n\targument + \"__service_role\" {\n\t\tcomment = \"Most annotation targets service or pod + that is all you want, however if the role is endpoints you we also want to consider + service annotations\"\n\t\toptional = true\n\t\tdefault = replace(coalesce(argument.role.value, + \"endpoints\"), \"endpoints\", \"service\")\n\t}\n\n\targument \"scrape_port_named_metrics\" + {\n\t\tcomment = \"Whether or not to automatically scrape endpoints that have + a port with 'metrics' in the name\"\n\t\toptional = true\n\t\tdefault = false\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: (.+))\"\n\t\toptional + = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment = \"A regex of metrics + to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" + {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + Targets From Docker Discovery\n\t*****************************************************************/\n\tdiscovery.kubernetes + \"annotation_metrics\" {\n\t\trole = coalesce(argument.role.value, \"endpoints\")\n\n\t\tselectors + {\n\t\t\trole = coalesce(argument.role.value, \"endpoints\")\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t*****************************************************************/\n\t// + filter metrics by kubernetes annotations\n\tdiscovery.relabel \"annotation_metrics_filter\" + {\n\t\ttargets = discovery.kubernetes.annotation_metrics.targets\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Targets to Keep or Drop\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare their metrics scraped or not\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/scrape: false\n\t\t//\n\t\t// the label prometheus.io/service-monitor: + \"false\" is a common label for headless services, when performing endpoint\n\t\t// + service discovery, if there is both a load-balanced service and headless service, + this can result in duplicate\n\t\t// scrapes if the name of the service is attached + as a label. any targets with this label or annotation set should be dropped\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"false\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, + \"endpoints\") + \"_label_prometheus_io_service_monitor\",\n\t\t\t]\n\t\t\tseparator + = \";\"\n\t\t\t// only allow empty or true, otherwise defaults to false\n\t\t\tregex + \ = \"^(?:;*)?(true)(;|true)*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\t// add a __tmp_scrape_port_named_metrics from + the argument.scrape_port_named_metrics\n\t\trule {\n\t\t\treplacement = format(\"%t\", + argument.scrape_port_named_metrics.value)\n\t\t\ttarget_label = \"__tmp_scrape_port_named_metrics\"\n\t\t}\n\n\t\t// + only keep targets that have scrape: true or \"metrics\" in the port name if the + argument scrape_port_named_metrics\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__tmp_scrape\",\n\t\t\t\t\"__tmp_scrape_port_named_metrics\",\n\t\t\t\t// + endpoints is the role and most meta labels started with \"endpoints\", however + the port name is an exception and starts with \"endpoint\"\n\t\t\t\t\"__meta_kubernetes_\" + + replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"endpoint\") + + \"_port_name\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(true;.*|(|true);true;(.*metrics.*))$\"\n\t\t}\n\n\t\t// + only keep targets where the pod is running or the pod_phase is empty and is not + an init container. This will only exist for role=\"pod\" or\n\t\t// potentially + role=\"endpoints\", if it is a service the value is empty and thus allowed to + pass, if it is an endpoint but not associated to a\n\t\t// pod but rather a static + IP or hostname, that could be outside of kubernetes allow endpoints to declare + what tenant their metrics should be\n\t\t// written to\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex + \ = \"^(?i)(Running|)$\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__meta_kubernetes_pod_ready\"]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\t\t// + if the container is an init container, drop it\n\t\trule {\n\t\t\taction = + \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_container_init\"]\n\t\t\tregex + \ = \"^(true)$\"\n\t\t}\n\n\t\t// allow resources to declare their metrics + the tenant their metrics should be sent to,\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/tenant: primary\n\t\t//\n\t\t// Note: This does not necessarily + have to be the actual tenantId, it can be a friendly name as well that is simply + used\n\t\t// to determine if the metrics should be gathered for the current + tenant\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_tenant\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_tenant\",\n\t\t\t]\n\t\t\tregex + = \"^(\" + coalesce(argument.tenant.value, \".*\") + \")$\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Scrape Metadata i.e. path, port, interval etc.\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare the protocol to use when collecting metrics, the default + value is \"http\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/scheme: + http\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"http\"\n\t\t\ttarget_label + = \"__scheme__\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scheme\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scheme\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?(https?).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scheme__\"\n\t\t}\n\n\t\t// allow resources + to declare the port to use when collecting metrics, the default value is the discovered + port from\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/port: 9090\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__address__\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_port\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_port\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^:]+)(?::\\\\d+)?;(\\\\d+)$\"\n\t\t\treplacement + \ = \"$1:$2\"\n\t\t\ttarget_label = \"__address__\"\n\t\t}\n\n\t\t// allow resources + to declare their the path to use when collecting their metrics, the default value + is \"/metrics\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/path: + /metrics/foo\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_path\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_path\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__metrics_path__\"\n\t\t}\n\n\t\t// allow resources + to declare how often their metrics should be collected, the default value is 1m,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/interval: 5m\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_interval\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_interval\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\t// allow + resources to declare the timeout of the scrape request, the default value is 10s,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/timeout: 30s\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_timeout\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_timeout\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Common Labels\n\t\t****************************************************************************************************************/\n\t\t// + set a source label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = \"kubernetes\"\n\t\t\ttarget_label = \"source\"\n\t\t}\n\n\t\t// set the cluster + label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = argument.cluster.value\n\t\t\ttarget_label + = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label + \ = \"namespace\"\n\t\t}\n\n\t\t// set the target name label i.e. service name, + pod name, etc.\n\t\t// if the role is endpoints, the first valued field is used + which would be __meta_kubernetes_pod_name, if the pod name is empty\n\t\t// then + the endpoint name would be used\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + argument.__pod_role.value + \"_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = + \"$1\"\n\t\t\ttarget_label = argument.__pod_role.value\n\t\t}\n\n\t\t// set a + default job label to be the namespace/pod_controller_name or namespace/service_name\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t\targument.__pod_role.value,\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^;]+)(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// if the controller + is a ReplicaSet, drop the hash from the end of the ReplicaSet\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_type\",\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:ReplicaSet);([^;]+);([^;]+)-.+$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// allow resources to + declare their the job label value to use when collecting their metrics, the default + value is \"\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/job: + integrations/kubernetes/cadvisor\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_job\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_job\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// set the app name if + specified as metadata labels \"app:\" or \"app.kubernetes.io/name:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"app\"\n\t\t}\n\n\t\t// + set the app component if specified as metadata labels \"component:\" or \"app.kubernetes.io/component:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_component\",\n\t\t\t]\n\t\t\tregex + \ = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"component\"\n\t\t}\n\n\t\t// set the version if specified as metadata labels + \"version:\" or \"app.kubernetes.io/version:\" or \"app_version:\"\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_version\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"version\"\n\t\t}\n\n\t\t// + set a workload label if the resource is a pod\n\t\t// example: grafana-agent-68nv9 + becomes DaemonSet/grafana-agent\n\t\trule {\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_kind\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"(.+);(.+)\"\n\t\t\treplacement = \"$1/$2\"\n\t\t\ttarget_label + = \"workload\"\n\t\t}\n\t\t// remove the hash from the ReplicaSet\n\t\trule {\n\t\t\tsource_labels + = [\"workload\"]\n\t\t\tregex = \"(ReplicaSet/.+)-.+\"\n\t\t\ttarget_label + \ = \"workload\"\n\t\t}\n\t}\n\n\t// only keep http targets\n\tdiscovery.relabel + \"http_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"http\"\n\t\t}\n\t}\n\n\t// only keep https targets\n\tdiscovery.relabel + \"https_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"https\"\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Scrape Labels Targets\n\t*****************************************************************/\n\t// + scrape http only targtetsa\n\tprometheus.scrape \"http_annotations\" {\n\t\ttargets + = discovery.relabel.http_annotations.output\n\n\t\tjob_name = \"annotation-metrics-http\"\n\t\tscheme + \ = \"http\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t// + scrape https only targtets\n\tprometheus.scrape \"https_annotations\" {\n\t\ttargets + = discovery.relabel.https_annotations.output\n\n\t\tjob_name = \"annotation-metrics-https\"\n\t\tscheme + \ = \"https\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\tbearer_token_file + = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n\n\t\ttls_config {\n\t\t\tca_file + \ = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n\t\t\tinsecure_skip_verify + = false\n\t\t\tserver_name = \"kubernetes\"\n\t\t}\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"annotations\" {\n\t\tforward_to = argument.forward_to.value\n\t\t// keep only + metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(.+)\")\n\t\t}\n\n\t\t// drop metrics that match the drop_metrics regex\n\t\trule + {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex + \ = coalesce(argument.drop_metrics.value, \"\")\n\t\t}\n\t}\n}\n" podmonitors-scrape.alloy: "/*\nModule Components: podmonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.podmonitors\n*/\n\ndeclare \"podmonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape PodMonitors\n\t*****************************************************************/\n\tprometheus.operator.podmonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.podmonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\t// selector {\n\t\t// \tmatch_expression {\n\t\t// \t\tkey = \"team\"\n\t\t// \t\toperator = \"In\"\n\t\t// \t\tvalues - \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n}\n" + \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"podmonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set the + cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" + rules-to-mimir.alloy: "/*\nModule Components: rules_to_mimir\nDescription: Auto + discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance.\n*/\n\ndeclare + \"rules_to_mimir\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + anonymous)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Mimir\n\t********************************************/\n\tmimir.rules.kubernetes + \"rules_to_mimir\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:8080\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\t// rule_selector {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= + \"true\",\n\t\t// \t}\n\t\t// }\n\t}\n}\n" servicemonitors-scrape.alloy: "/*\nModule Components: servicemonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.servicemonitors\n*/\n\ndeclare \"servicemonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape ServiceMonitors\n\t*****************************************************************/\n\tprometheus.operator.servicemonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.servicemonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering - {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n" + {\n\t\t\tenabled = true\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"servicemonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set + the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f namespace: monitoring-system --- apiVersion: v1 @@ -405,19 +751,21 @@ data: \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(ProfilessReceiver) where collected - logs should be forwarded to\"\n\t}\n\n\tdiscovery.kubernetes \"pyroscope_kubernetes\" - {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape config allows to define - annotations based scraping.\n\t//\n\t// For example the following annotations:\n\t//\n\t// - ```\n\t// profiles.grafana.com/memory.scrape: \"true\"\n\t// profiles.grafana.com/memory.port: - \"8080\"\n\t// profiles.grafana.com/cpu.scrape: \"true\"\n\t// profiles.grafana.com/cpu.port: - \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: \"true\"\n\t// profiles.grafana.com/goroutine.port: - \"8080\"\n\t// ```\n\t//\n\t// will scrape the `memory`, `cpu` and `goroutine` - profiles from the `8080` port of the pod.\n\t//\n\t// For more information see - https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel + logs should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\tdiscovery.kubernetes + \"pyroscope_kubernetes\" {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape + config allows to define annotations based scraping.\n\t//\n\t// For example the + following annotations:\n\t//\n\t// ```\n\t// profiles.grafana.com/memory.scrape: + \"true\"\n\t// profiles.grafana.com/memory.port: \"8080\"\n\t// profiles.grafana.com/cpu.scrape: + \"true\"\n\t// profiles.grafana.com/cpu.port: \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: + \"true\"\n\t// profiles.grafana.com/goroutine.port: \"8080\"\n\t// ```\n\t//\n\t// + will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port + of the pod.\n\t//\n\t// For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel \"kubernetes_pods\" {\n\t\ttargets = concat(discovery.kubernetes.pyroscope_kubernetes.targets)\n\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex \ = \"Pending|Succeeded|Failed|Completed\"\n\t\t}\n\n\t\trule {\n\t\t\taction - = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\trule + = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\trule @@ -636,7 +984,7 @@ data: {\n\t\t\t\tenabled = true\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 namespace: monitoring-system --- apiVersion: v1 @@ -646,40 +994,58 @@ data: Transformation\ndeclare \"process_and_transform\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"traces_forward_to\" {\n\t\tcomment = \"Must be a list(TracesReceiver) where - collected traces should be forwarded to\"\n\t}\n\n\targument \"logs_forward_to\" - {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be - forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment = \"Must - be a list(MetricsReceiver) where collected metrics should be forwarded to\"\n\t}\n\n\targument - \"cluster\" {\n\t\toptional = true\n\t\tdefault = \"k3d-k3s-codelab\"\n\t}\n\n\targument - \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4318\"\n\t}\n\n\targument - \"otlp_grpc_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* + collected traces should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\targument + \"logs_forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected + logs should be forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment + = \"Must be a list(MetricsReceiver) where collected metrics should be forwarded + to\"\n\t}\n\n\targument \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault + \ = \"0.0.0.0:4318\"\n\t}\n\n\targument \"otlp_grpc_endpoint\" {\n\t\toptional + = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* Jaeger for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.jaeger \"default\" {\n\t\tprotocols {\n\t\t\tgrpc {\n\t\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t\t}\n\n\t\t\tthrift_http {\n\t\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t\t}\n\n\t\t\tthrift_binary {\n\t\t\t\tendpoint = \"0.0.0.0:6832\"\n\t\t\t}\n\n\t\t\tthrift_compact {\n\t\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [otelcol.processor.resourcedetection.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + {\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* Otelcol for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.otlp - \"default\" {\n\t\tgrpc {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp - {\n\t\t\tendpoint = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics - = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection - \"default\" {\n\t\tdetectors = [\"env\"]\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces - = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes + \"default\" {\n\t\tdebug_metrics {\n\t\t\tdisable_high_cardinality_metrics = true\n\t\t}\n\n\t\tgrpc + {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp {\n\t\t\tendpoint + = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.resourcedetection.default.input]\n\t\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection + \"default\" {\n\t\tdetectors = [\"env\", \"system\"]\n\n\t\tsystem {\n\t\t\thostname_sources + = [\"os\"]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\n\t\t\tlogs + \ = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform + \"add_metric_datapoint_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements + {\n\t\t\tcontext = \"datapoint\"\n\t\t\tstatements = [\n\t\t\t\t\"set(attributes[\\\"deployment.environment\\\"], + resource.attributes[\\\"deployment.environment\\\"])\",\n\t\t\t\t\"set(attributes[\\\"service.version\\\"], + resource.attributes[\\\"service.version\\\"])\",\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes \"default\" {\n\t\textract {\n\t\t\tmetadata = [\n\t\t\t\t\"k8s.namespace.name\",\n\t\t\t\t\"k8s.pod.name\",\n\t\t\t\t\"k8s.deployment.name\",\n\t\t\t\t\"k8s.statefulset.name\",\n\t\t\t\t\"k8s.daemonset.name\",\n\t\t\t\t\"k8s.cronjob.name\",\n\t\t\t\t\"k8s.job.name\",\n\t\t\t\t\"k8s.node.name\",\n\t\t\t\t\"k8s.pod.uid\",\n\t\t\t\t\"k8s.pod.start_time\",\n\t\t\t]\n\t\t}\n\n\t\tpod_association {\n\t\t\tsource {\n\t\t\t\tfrom = \"connection\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t\ttraces - = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform - \"add_resource_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tlog_statements + {\n\t\t\tmetrics = [otelcol.processor.transform.default.input]\n\t\t\tlogs = + [otelcol.processor.transform.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.transform.default.input,\n\t\t\t\totelcol.connector.host_info.default.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.connector.host_info + \"default\" {\n\t\thost_identifiers = [\"k8s.node.name\"]\n\n\t\toutput {\n\t\t\tmetrics + = [otelcol.processor.batch.host_info_batch.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + \"host_info_batch\" {\n\t\toutput {\n\t\t\tmetrics = [otelcol.exporter.prometheus.host_info_metrics.input]\n\t\t}\n\t}\n\n\totelcol.exporter.prometheus + \"host_info_metrics\" {\n\t\tadd_metric_suffixes = false\n\t\tforward_to = + argument.metrics_forward_to.value\n\t}\n\n\totelcol.processor.transform \"default\" + {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements {\n\t\t\tcontext = + \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\tlog_statements {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])`,\n\t\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t\t`set(attributes[\"loki.resource.labels\"], \"pod, namespace, cluster, job\")`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements - {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], - \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter - \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.batch.default.input]\n\t\t\ttraces - = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements + {\n\t\t\tcontext = \"span\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces + \ = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter + \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\ttraces {\n\t\t\tspan = [\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/live\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] == \\\"/healthy\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/ready\\\"\",\n\t\t\t]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs + \ = [otelcol.processor.batch.default.input]\n\t\t\ttraces = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch \"default\" {\n\t\tsend_batch_size = 16384\n\t\tsend_batch_max_size = 0\n\t\ttimeout \ = \"5s\"\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\t\tlogs \ = [otelcol.processor.memory_limiter.default.input]\n\t\t\ttraces = [otelcol.processor.memory_limiter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.memory_limiter @@ -715,7 +1081,7 @@ data: \"alloy_traces_input\" {\n\t\tvalue = otelcol.processor.batch.default.input\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 namespace: monitoring-system --- apiVersion: v1 @@ -1330,19 +1696,19 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-56888m5g8t + name: alloy-config-m5kk6k2f7k name: config - configMap: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f name: modules-kubernetes-metrics - configMap: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k name: modules-kubernetes-logs - configMap: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 name: modules-kubernetes-traces - configMap: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 name: modules-kubernetes-profiles --- apiVersion: apps/v1 diff --git a/kubernetes/monolithic-mode/profiles/profiles.alloy b/kubernetes/monolithic-mode/profiles/profiles.alloy index cf0354fa..8828dd5b 100644 --- a/kubernetes/monolithic-mode/profiles/profiles.alloy +++ b/kubernetes/monolithic-mode/profiles/profiles.alloy @@ -26,7 +26,8 @@ import.file "profiles" { } profiles.annotations_scrape "kubernetes" { - // annotation_prefix = "profiles.grafana.com" + cluster = "k3d-k3s-codelab" + forward_to = [provider.self_hosted_stack.kubernetes.profiles_receiver] } @@ -37,10 +38,16 @@ import.file "metrics" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/metrics" } -metrics.podmonitors_scrape "kubernetes" { +metrics.annotations_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } metrics.servicemonitors_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } diff --git a/kubernetes/monolithic-mode/traces/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/traces/k8s-all-in-one.yaml index 0a42722f..e04d9aaa 100644 --- a/kubernetes/monolithic-mode/traces/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/traces/k8s-all-in-one.yaml @@ -225,47 +225,47 @@ data: * Traces\n ********************************************/\nimport.file \"traces\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/traces\"\n}\n\n// traces Processing And Transformation process_and_transform\ntraces.process_and_transform - \"kubernetes\" {\n\tmetrics_forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\tlogs_forward_to + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\n\tmetrics_forward_to + = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\tlogs_forward_to \ = [provider.self_hosted_stack.kubernetes.logs_receiver]\n\ttraces_forward_to \ = [provider.self_hosted_stack.kubernetes.traces_receiver]\n}\n\ntracing {\n\t// Write all spans. Don't do this in production!\n\tsampling_fraction = 1\n\n\t// Forward Alloy internal spans to traces process.\n\twrite_to = [traces.process_and_transform.kubernetes.alloy_traces_input]\n}\n\n/********************************************\n * Metrics\n ********************************************/\nimport.file \"metrics\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/metrics\"\n}\n\nmetrics.podmonitors_scrape \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape - \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" + + \"/kubernetes/metrics\"\n}\n\nmetrics.annotations_scrape \"kubernetes\" {\n\tcluster + \ = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-t69dk2g68h + name: alloy-config-284mhm8bfc namespace: monitoring-system --- apiVersion: v1 data: annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: Scrapes targets for logs based on kubernetes Pod annotations\n\n Annotations:\n - \ logs.grafana.com/ingest: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare + \ logs.grafana.com/scrape: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this - can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\t// arguments - for kubernetes discovery\n\targument \"namespaces\" {\n\t\tcomment = \"The namespaces - to look for targets in (default: [\\\"kube-system\\\"] is all namespaces)\"\n\t\toptional - = true\n\t}\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\targument + \"cluster\" { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix to use (default: logs.grafana.com)\"\n\t\tdefault = \"logs.grafana.com\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional = true\n\t\tcomment \ = \"The logic is used to transform the annotation argument into a valid label name by removing unsupported characters.\"\n\t\tdefault = replace(replace(replace(coalesce(argument.annotation_prefix.value, \"logs.grafana.com\"), \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\t// - find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\n\t\tnamespaces - {\n\t\t\tnames = coalesce(argument.namespaces.value, [])\n\t\t}\n\t}\n\n\t// filter - logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" + find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\t}\n\n\t// + filter logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" {\n\t\ttargets = discovery.kubernetes.annotation_logs.targets\n\n\t\t// allow pods to declare their logs to be ingested or not, the default is true\n\t\t// - \ i.e. logs.grafana.com/ingest: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + \ i.e. logs.grafana.com/scrape: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\n\t\t// allow pods to declare what tenant their logs should be written to, the following annotation @@ -275,6 +275,8 @@ data: + argument.tenant.value + \")$\"\n\t\t}\n\n\t\t// set the instance label as the name of the worker node the pod is on\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_node_name\"]\n\t\t\ttarget_label = \"instance\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\t// set the pod label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\t// @@ -339,12 +341,11 @@ data: ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\toptional - = true\n\t}\n\n\targument \"cluster\" {\n\t\toptional = true\n\t}\n\n\tloki.source.kubernetes_events - \"cluster_events\" {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format + = true\n\t}\n\n\targument \"cluster\" { }\n\n\tloki.source.kubernetes_events \"cluster_events\" + {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format = \"logfmt\"\n\t\tforward_to = [loki.process.logs_service.receiver]\n\t}\n\n\tloki.process \"logs_service\" {\n\t\tstage.static_labels {\n\t\t\tvalues = {\n\t\t\t\tcluster - = coalesce(argument.cluster.value, \"k3d\"),\n\t\t\t}\n\t\t}\n\t\tforward_to = - argument.forward_to.value\n\t}\n}\n" + = argument.cluster.value,\n\t\t\t}\n\t\t}\n\t\tforward_to = argument.forward_to.value\n\t}\n}\n" keep-labels.alloy: "/*\nModule Components: keep_labels\nDescription: Pre-defined set of labels to keep, this stage should always be in-place as the previous relabeing\n \ stages make every pod label and annotation a label in the pipeline, @@ -366,46 +367,392 @@ data: = argument.keep_labels.value\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* EXPORTS\n\t*****************************************************************/\n\texport \"receiver\" {\n\t\tvalue = loki.process.keep_labels.receiver\n\t}\n}\n" + rules-to-loki.alloy: "/*\nModule Components: rules_to_loki\nDescription: Auto discovers + PrometheusRule Kubernetes resources and loads them into a Loki instance.\n*/\n\ndeclare + \"rules_to_loki\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + fake)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Loki\n\t********************************************/\n\tloki.rules.kubernetes + \"rules_to_loki\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:3100\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_loki= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\trule_selector {\n\t\t\tmatch_labels = {\n\t\t\t\tauto_rules_to_loki + = \"true\",\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k namespace: monitoring-system --- apiVersion: v1 data: + annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: + Scrapes targets for metrics based on kubernetes annotations\n\nNote: Every argument + except for \"forward_to\" is optional, and does have a defined default value. + \ However, the values for these\n arguments are not defined using the default + = \" ... \" argument syntax, but rather using the coalesce(argument.value, \" + ... \").\n This is because if the argument passed in from another consuming + module is set to null, the default = \" ... \" syntax will\n does not override + the value passed in, where coalesce() will return the first non-null value.\n\nKubernetes + Annotation Auto-Scraping\n------------------------------------------------------------------------------------------------------------------------------------\nThis + module is meant to be used to automatically scrape targets based on a certain + role and set of annotations. This module can be consumed\nmultiple times with + different roles. The supported roles are:\n\n - pod\n - service\n - endpoints\n\nTypically, + if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality + you would use role=\"endpoints\", if\nmimicking the behavior of the PodMonitor + functionality you would use role=\"pod\". It is important to understand that + with endpoints,\nthe target is typically going to be a pod, and whatever annotations + that are set on the service will automatically be propagated to the\nendpoints. + \ This is why the role \"endpoints\" is used, because it will scrape the pod, + but also consider the service annotations. Using\nrole=\"endpoints\", which scrape + each endpoint associated to the service. If role=\"service\" is used, it will + only scrape the service, only\nhitting one of the endpoints associated to the + service.\n\nThis is where you must consider your scraping strategy, for example + if you scrape a service like \"kube-state-metrics\" using\nrole=\"endpoints\" + you should only have a single replica of the kube-state-metrics pod, if you have + multiple replicas, you should use\nrole=\"service\" or a separate non-annotation + job completely. Scraping a service instead of endpoints, is typically a rare + use case, but\nit is supported.\n\nThere are other considerations for using annotation + based scraping as well, which is metric relabeling rules that happen post scrape. + \ If\nyou have a target that you want to apply a bunch of relabelings to or a + very large metrics response payload, performance wise it will be\nbetter to have + a separate job for that target, rather than using use annotations. As every targert + will go through the ssame relabeling.\nTypical deployment strategies/options would + be:\n\nOption #1 (recommended):\n - Annotation Scraping for role=\"endpoints\"\n + \ - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, + etc.) or large metric payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, + kube-apiserver, kube-scheduler, etc.)\n\nOption #2:\n - Annotation Scraping for + role=\"pod\"\n - Annotation Scraping for role=\"service\" (i.e. kube-state-metrics, + node-exporter, etc.)\n - Separate Jobs for specific use cases or large metric + payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, + kube-scheduler, etc.)\n\nAt no point should you use role=\"endpoints\" and role=\"pod\" + together, as this will result in duplicate targets being scraped, thus\ngenerating + duplicate metrics. If you want to scrape both the pod and the service, use Option + #2.\n\nEach port attached to an service/pod/endpoint is an eligible target, oftentimes + it will have multiple ports.\nThere may be instances when you want to scrape all + ports or some ports and not others. To support this\nthe following annotations + are available:\n\n metrics.grafana.com/scrape: true\n\nthe default scraping scheme + is http, this can be specified as a single value which would override, the schema + being used for all\nports attached to the target:\n\n metrics.grafana.com/scheme: + https\n\nthe default path to scrape is /metrics, this can be specified as a single + value which would override, the scrape path being used\nfor all ports attached + to the target:\n\n metrics.grafana.com/path: /metrics/some_path\n\nthe default + port to scrape is the target port, this can be specified as a single value which + would override the scrape port being\nused for all ports attached to the target, + note that even if aan target had multiple targets, the relabel_config targets + are\ndeduped before scraping:\n\n metrics.grafana.com/port: 8080\n\nthe default + interval to scrape is 1m, this can be specified as a single value which would + override, the scrape interval being used\nfor all ports attached to the target:\n\n + \ metrics.grafana.com/interval: 5m\n\nthe default timeout for scraping is 10s, + this can be specified as a single value which would override, the scrape interval + being\nused for all ports attached to the target:\n\n metrics.grafana.com/timeout: + 30s\n\nthe default job is namespace/{{ service name }} or namespace/{{ controller_name + }} depending on the role, there may be instances\nin which a different job name + is required because of a set of dashboards, rules, etc. to support this there + is a job annotation\nwhich will override the default value:\n\n metrics.grafana.com/job: + integrations/kubernetes/kube-state-metrics\n*/\n\ndeclare \"annotations_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment + \ = \"The tenant to filter metrics to. This does not have to be the tenantId, + this is the value to look for in the metrics.agent.grafana.com/tenant annotation, + and this can be a regex.\"\n\t\toptional = true\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + to use (default: metrics.grafana.com)\"\n\t\tdefault = \"metrics.grafana.com\"\n\t\toptional + = true\n\t}\n\n\targument \"role\" {\n\t\tcomment = \"The role to use when looking + for targets to scrape via annotations, can be: endpoints, service, pod (default: + endpoints)\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional + = true\n\t\tcomment = \"The logic is used to transform the annotation argument + into a valid label name by removing unsupported characters.\"\n\t\tdefault = + replace(replace(replace(coalesce(argument.annotation_prefix.value, \"metrics.grafana.com\"), + \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\targument \"__pod_role\" + {\n\t\tcomment = \"Most annotation targets service or pod that is all you want, + however if the role is endpoints you want the pod\"\n\t\toptional = true\n\t\tdefault + \ = replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"pod\")\n\t}\n\n\targument + \"__service_role\" {\n\t\tcomment = \"Most annotation targets service or pod + that is all you want, however if the role is endpoints you we also want to consider + service annotations\"\n\t\toptional = true\n\t\tdefault = replace(coalesce(argument.role.value, + \"endpoints\"), \"endpoints\", \"service\")\n\t}\n\n\targument \"scrape_port_named_metrics\" + {\n\t\tcomment = \"Whether or not to automatically scrape endpoints that have + a port with 'metrics' in the name\"\n\t\toptional = true\n\t\tdefault = false\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: (.+))\"\n\t\toptional + = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment = \"A regex of metrics + to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" + {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + Targets From Docker Discovery\n\t*****************************************************************/\n\tdiscovery.kubernetes + \"annotation_metrics\" {\n\t\trole = coalesce(argument.role.value, \"endpoints\")\n\n\t\tselectors + {\n\t\t\trole = coalesce(argument.role.value, \"endpoints\")\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t*****************************************************************/\n\t// + filter metrics by kubernetes annotations\n\tdiscovery.relabel \"annotation_metrics_filter\" + {\n\t\ttargets = discovery.kubernetes.annotation_metrics.targets\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Targets to Keep or Drop\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare their metrics scraped or not\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/scrape: false\n\t\t//\n\t\t// the label prometheus.io/service-monitor: + \"false\" is a common label for headless services, when performing endpoint\n\t\t// + service discovery, if there is both a load-balanced service and headless service, + this can result in duplicate\n\t\t// scrapes if the name of the service is attached + as a label. any targets with this label or annotation set should be dropped\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"false\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, + \"endpoints\") + \"_label_prometheus_io_service_monitor\",\n\t\t\t]\n\t\t\tseparator + = \";\"\n\t\t\t// only allow empty or true, otherwise defaults to false\n\t\t\tregex + \ = \"^(?:;*)?(true)(;|true)*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\t// add a __tmp_scrape_port_named_metrics from + the argument.scrape_port_named_metrics\n\t\trule {\n\t\t\treplacement = format(\"%t\", + argument.scrape_port_named_metrics.value)\n\t\t\ttarget_label = \"__tmp_scrape_port_named_metrics\"\n\t\t}\n\n\t\t// + only keep targets that have scrape: true or \"metrics\" in the port name if the + argument scrape_port_named_metrics\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__tmp_scrape\",\n\t\t\t\t\"__tmp_scrape_port_named_metrics\",\n\t\t\t\t// + endpoints is the role and most meta labels started with \"endpoints\", however + the port name is an exception and starts with \"endpoint\"\n\t\t\t\t\"__meta_kubernetes_\" + + replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"endpoint\") + + \"_port_name\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(true;.*|(|true);true;(.*metrics.*))$\"\n\t\t}\n\n\t\t// + only keep targets where the pod is running or the pod_phase is empty and is not + an init container. This will only exist for role=\"pod\" or\n\t\t// potentially + role=\"endpoints\", if it is a service the value is empty and thus allowed to + pass, if it is an endpoint but not associated to a\n\t\t// pod but rather a static + IP or hostname, that could be outside of kubernetes allow endpoints to declare + what tenant their metrics should be\n\t\t// written to\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex + \ = \"^(?i)(Running|)$\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__meta_kubernetes_pod_ready\"]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\t\t// + if the container is an init container, drop it\n\t\trule {\n\t\t\taction = + \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_container_init\"]\n\t\t\tregex + \ = \"^(true)$\"\n\t\t}\n\n\t\t// allow resources to declare their metrics + the tenant their metrics should be sent to,\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/tenant: primary\n\t\t//\n\t\t// Note: This does not necessarily + have to be the actual tenantId, it can be a friendly name as well that is simply + used\n\t\t// to determine if the metrics should be gathered for the current + tenant\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_tenant\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_tenant\",\n\t\t\t]\n\t\t\tregex + = \"^(\" + coalesce(argument.tenant.value, \".*\") + \")$\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Scrape Metadata i.e. path, port, interval etc.\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare the protocol to use when collecting metrics, the default + value is \"http\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/scheme: + http\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"http\"\n\t\t\ttarget_label + = \"__scheme__\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scheme\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scheme\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?(https?).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scheme__\"\n\t\t}\n\n\t\t// allow resources + to declare the port to use when collecting metrics, the default value is the discovered + port from\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/port: 9090\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__address__\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_port\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_port\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^:]+)(?::\\\\d+)?;(\\\\d+)$\"\n\t\t\treplacement + \ = \"$1:$2\"\n\t\t\ttarget_label = \"__address__\"\n\t\t}\n\n\t\t// allow resources + to declare their the path to use when collecting their metrics, the default value + is \"/metrics\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/path: + /metrics/foo\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_path\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_path\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__metrics_path__\"\n\t\t}\n\n\t\t// allow resources + to declare how often their metrics should be collected, the default value is 1m,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/interval: 5m\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_interval\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_interval\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\t// allow + resources to declare the timeout of the scrape request, the default value is 10s,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/timeout: 30s\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_timeout\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_timeout\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Common Labels\n\t\t****************************************************************************************************************/\n\t\t// + set a source label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = \"kubernetes\"\n\t\t\ttarget_label = \"source\"\n\t\t}\n\n\t\t// set the cluster + label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = argument.cluster.value\n\t\t\ttarget_label + = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label + \ = \"namespace\"\n\t\t}\n\n\t\t// set the target name label i.e. service name, + pod name, etc.\n\t\t// if the role is endpoints, the first valued field is used + which would be __meta_kubernetes_pod_name, if the pod name is empty\n\t\t// then + the endpoint name would be used\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + argument.__pod_role.value + \"_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = + \"$1\"\n\t\t\ttarget_label = argument.__pod_role.value\n\t\t}\n\n\t\t// set a + default job label to be the namespace/pod_controller_name or namespace/service_name\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t\targument.__pod_role.value,\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^;]+)(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// if the controller + is a ReplicaSet, drop the hash from the end of the ReplicaSet\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_type\",\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:ReplicaSet);([^;]+);([^;]+)-.+$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// allow resources to + declare their the job label value to use when collecting their metrics, the default + value is \"\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/job: + integrations/kubernetes/cadvisor\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_job\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_job\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// set the app name if + specified as metadata labels \"app:\" or \"app.kubernetes.io/name:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"app\"\n\t\t}\n\n\t\t// + set the app component if specified as metadata labels \"component:\" or \"app.kubernetes.io/component:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_component\",\n\t\t\t]\n\t\t\tregex + \ = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"component\"\n\t\t}\n\n\t\t// set the version if specified as metadata labels + \"version:\" or \"app.kubernetes.io/version:\" or \"app_version:\"\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_version\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"version\"\n\t\t}\n\n\t\t// + set a workload label if the resource is a pod\n\t\t// example: grafana-agent-68nv9 + becomes DaemonSet/grafana-agent\n\t\trule {\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_kind\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"(.+);(.+)\"\n\t\t\treplacement = \"$1/$2\"\n\t\t\ttarget_label + = \"workload\"\n\t\t}\n\t\t// remove the hash from the ReplicaSet\n\t\trule {\n\t\t\tsource_labels + = [\"workload\"]\n\t\t\tregex = \"(ReplicaSet/.+)-.+\"\n\t\t\ttarget_label + \ = \"workload\"\n\t\t}\n\t}\n\n\t// only keep http targets\n\tdiscovery.relabel + \"http_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"http\"\n\t\t}\n\t}\n\n\t// only keep https targets\n\tdiscovery.relabel + \"https_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"https\"\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Scrape Labels Targets\n\t*****************************************************************/\n\t// + scrape http only targtetsa\n\tprometheus.scrape \"http_annotations\" {\n\t\ttargets + = discovery.relabel.http_annotations.output\n\n\t\tjob_name = \"annotation-metrics-http\"\n\t\tscheme + \ = \"http\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t// + scrape https only targtets\n\tprometheus.scrape \"https_annotations\" {\n\t\ttargets + = discovery.relabel.https_annotations.output\n\n\t\tjob_name = \"annotation-metrics-https\"\n\t\tscheme + \ = \"https\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\tbearer_token_file + = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n\n\t\ttls_config {\n\t\t\tca_file + \ = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n\t\t\tinsecure_skip_verify + = false\n\t\t\tserver_name = \"kubernetes\"\n\t\t}\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"annotations\" {\n\t\tforward_to = argument.forward_to.value\n\t\t// keep only + metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(.+)\")\n\t\t}\n\n\t\t// drop metrics that match the drop_metrics regex\n\t\trule + {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex + \ = coalesce(argument.drop_metrics.value, \"\")\n\t\t}\n\t}\n}\n" podmonitors-scrape.alloy: "/*\nModule Components: podmonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.podmonitors\n*/\n\ndeclare \"podmonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape PodMonitors\n\t*****************************************************************/\n\tprometheus.operator.podmonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.podmonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\t// selector {\n\t\t// \tmatch_expression {\n\t\t// \t\tkey = \"team\"\n\t\t// \t\toperator = \"In\"\n\t\t// \t\tvalues - \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n}\n" + \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"podmonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set the + cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" + rules-to-mimir.alloy: "/*\nModule Components: rules_to_mimir\nDescription: Auto + discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance.\n*/\n\ndeclare + \"rules_to_mimir\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + anonymous)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Mimir\n\t********************************************/\n\tmimir.rules.kubernetes + \"rules_to_mimir\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:8080\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\t// rule_selector {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= + \"true\",\n\t\t// \t}\n\t\t// }\n\t}\n}\n" servicemonitors-scrape.alloy: "/*\nModule Components: servicemonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.servicemonitors\n*/\n\ndeclare \"servicemonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape ServiceMonitors\n\t*****************************************************************/\n\tprometheus.operator.servicemonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.servicemonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering - {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n" + {\n\t\t\tenabled = true\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"servicemonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set + the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f namespace: monitoring-system --- apiVersion: v1 @@ -415,19 +762,21 @@ data: \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(ProfilessReceiver) where collected - logs should be forwarded to\"\n\t}\n\n\tdiscovery.kubernetes \"pyroscope_kubernetes\" - {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape config allows to define - annotations based scraping.\n\t//\n\t// For example the following annotations:\n\t//\n\t// - ```\n\t// profiles.grafana.com/memory.scrape: \"true\"\n\t// profiles.grafana.com/memory.port: - \"8080\"\n\t// profiles.grafana.com/cpu.scrape: \"true\"\n\t// profiles.grafana.com/cpu.port: - \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: \"true\"\n\t// profiles.grafana.com/goroutine.port: - \"8080\"\n\t// ```\n\t//\n\t// will scrape the `memory`, `cpu` and `goroutine` - profiles from the `8080` port of the pod.\n\t//\n\t// For more information see - https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel + logs should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\tdiscovery.kubernetes + \"pyroscope_kubernetes\" {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape + config allows to define annotations based scraping.\n\t//\n\t// For example the + following annotations:\n\t//\n\t// ```\n\t// profiles.grafana.com/memory.scrape: + \"true\"\n\t// profiles.grafana.com/memory.port: \"8080\"\n\t// profiles.grafana.com/cpu.scrape: + \"true\"\n\t// profiles.grafana.com/cpu.port: \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: + \"true\"\n\t// profiles.grafana.com/goroutine.port: \"8080\"\n\t// ```\n\t//\n\t// + will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port + of the pod.\n\t//\n\t// For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel \"kubernetes_pods\" {\n\t\ttargets = concat(discovery.kubernetes.pyroscope_kubernetes.targets)\n\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex \ = \"Pending|Succeeded|Failed|Completed\"\n\t\t}\n\n\t\trule {\n\t\t\taction - = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\trule + = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\trule @@ -646,7 +995,7 @@ data: {\n\t\t\t\tenabled = true\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 namespace: monitoring-system --- apiVersion: v1 @@ -656,40 +1005,58 @@ data: Transformation\ndeclare \"process_and_transform\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"traces_forward_to\" {\n\t\tcomment = \"Must be a list(TracesReceiver) where - collected traces should be forwarded to\"\n\t}\n\n\targument \"logs_forward_to\" - {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be - forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment = \"Must - be a list(MetricsReceiver) where collected metrics should be forwarded to\"\n\t}\n\n\targument - \"cluster\" {\n\t\toptional = true\n\t\tdefault = \"k3d-k3s-codelab\"\n\t}\n\n\targument - \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4318\"\n\t}\n\n\targument - \"otlp_grpc_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* + collected traces should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\targument + \"logs_forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected + logs should be forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment + = \"Must be a list(MetricsReceiver) where collected metrics should be forwarded + to\"\n\t}\n\n\targument \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault + \ = \"0.0.0.0:4318\"\n\t}\n\n\targument \"otlp_grpc_endpoint\" {\n\t\toptional + = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* Jaeger for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.jaeger \"default\" {\n\t\tprotocols {\n\t\t\tgrpc {\n\t\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t\t}\n\n\t\t\tthrift_http {\n\t\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t\t}\n\n\t\t\tthrift_binary {\n\t\t\t\tendpoint = \"0.0.0.0:6832\"\n\t\t\t}\n\n\t\t\tthrift_compact {\n\t\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [otelcol.processor.resourcedetection.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + {\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* Otelcol for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.otlp - \"default\" {\n\t\tgrpc {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp - {\n\t\t\tendpoint = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics - = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection - \"default\" {\n\t\tdetectors = [\"env\"]\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces - = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes + \"default\" {\n\t\tdebug_metrics {\n\t\t\tdisable_high_cardinality_metrics = true\n\t\t}\n\n\t\tgrpc + {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp {\n\t\t\tendpoint + = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.resourcedetection.default.input]\n\t\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection + \"default\" {\n\t\tdetectors = [\"env\", \"system\"]\n\n\t\tsystem {\n\t\t\thostname_sources + = [\"os\"]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\n\t\t\tlogs + \ = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform + \"add_metric_datapoint_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements + {\n\t\t\tcontext = \"datapoint\"\n\t\t\tstatements = [\n\t\t\t\t\"set(attributes[\\\"deployment.environment\\\"], + resource.attributes[\\\"deployment.environment\\\"])\",\n\t\t\t\t\"set(attributes[\\\"service.version\\\"], + resource.attributes[\\\"service.version\\\"])\",\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes \"default\" {\n\t\textract {\n\t\t\tmetadata = [\n\t\t\t\t\"k8s.namespace.name\",\n\t\t\t\t\"k8s.pod.name\",\n\t\t\t\t\"k8s.deployment.name\",\n\t\t\t\t\"k8s.statefulset.name\",\n\t\t\t\t\"k8s.daemonset.name\",\n\t\t\t\t\"k8s.cronjob.name\",\n\t\t\t\t\"k8s.job.name\",\n\t\t\t\t\"k8s.node.name\",\n\t\t\t\t\"k8s.pod.uid\",\n\t\t\t\t\"k8s.pod.start_time\",\n\t\t\t]\n\t\t}\n\n\t\tpod_association {\n\t\t\tsource {\n\t\t\t\tfrom = \"connection\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t\ttraces - = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform - \"add_resource_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tlog_statements + {\n\t\t\tmetrics = [otelcol.processor.transform.default.input]\n\t\t\tlogs = + [otelcol.processor.transform.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.transform.default.input,\n\t\t\t\totelcol.connector.host_info.default.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.connector.host_info + \"default\" {\n\t\thost_identifiers = [\"k8s.node.name\"]\n\n\t\toutput {\n\t\t\tmetrics + = [otelcol.processor.batch.host_info_batch.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + \"host_info_batch\" {\n\t\toutput {\n\t\t\tmetrics = [otelcol.exporter.prometheus.host_info_metrics.input]\n\t\t}\n\t}\n\n\totelcol.exporter.prometheus + \"host_info_metrics\" {\n\t\tadd_metric_suffixes = false\n\t\tforward_to = + argument.metrics_forward_to.value\n\t}\n\n\totelcol.processor.transform \"default\" + {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements {\n\t\t\tcontext = + \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\tlog_statements {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])`,\n\t\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t\t`set(attributes[\"loki.resource.labels\"], \"pod, namespace, cluster, job\")`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements - {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], - \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter - \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.batch.default.input]\n\t\t\ttraces - = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements + {\n\t\t\tcontext = \"span\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces + \ = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter + \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\ttraces {\n\t\t\tspan = [\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/live\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] == \\\"/healthy\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/ready\\\"\",\n\t\t\t]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs + \ = [otelcol.processor.batch.default.input]\n\t\t\ttraces = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch \"default\" {\n\t\tsend_batch_size = 16384\n\t\tsend_batch_max_size = 0\n\t\ttimeout \ = \"5s\"\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\t\tlogs \ = [otelcol.processor.memory_limiter.default.input]\n\t\t\ttraces = [otelcol.processor.memory_limiter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.memory_limiter @@ -725,7 +1092,7 @@ data: \"alloy_traces_input\" {\n\t\tvalue = otelcol.processor.batch.default.input\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 namespace: monitoring-system --- apiVersion: v1 @@ -1420,19 +1787,19 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-t69dk2g68h + name: alloy-config-284mhm8bfc name: config - configMap: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f name: modules-kubernetes-metrics - configMap: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k name: modules-kubernetes-logs - configMap: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 name: modules-kubernetes-traces - configMap: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 name: modules-kubernetes-profiles --- apiVersion: apps/v1 diff --git a/kubernetes/monolithic-mode/traces/traces.alloy b/kubernetes/monolithic-mode/traces/traces.alloy index cbae15b5..42f255e1 100644 --- a/kubernetes/monolithic-mode/traces/traces.alloy +++ b/kubernetes/monolithic-mode/traces/traces.alloy @@ -28,6 +28,8 @@ import.file "traces" { // traces Processing And Transformation process_and_transform traces.process_and_transform "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + metrics_forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] logs_forward_to = [provider.self_hosted_stack.kubernetes.logs_receiver] traces_forward_to = [provider.self_hosted_stack.kubernetes.traces_receiver] @@ -48,10 +50,16 @@ import.file "metrics" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/metrics" } -metrics.podmonitors_scrape "kubernetes" { +metrics.annotations_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } metrics.servicemonitors_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } diff --git a/kubernetes/read-write-mode/logs/k8s-all-in-one.yaml b/kubernetes/read-write-mode/logs/k8s-all-in-one.yaml index 83d7340b..74cca8e0 100644 --- a/kubernetes/read-write-mode/logs/k8s-all-in-one.yaml +++ b/kubernetes/read-write-mode/logs/k8s-all-in-one.yaml @@ -373,31 +373,27 @@ data: = coalesce(env(\"SELF_HOSTED_METRICS_ENDPOINT_URL\"), \"http://nginx.gateway.svc:8080/api/v1/push\")\n\tlogs_endpoint_url \ = coalesce(env(\"SELF_HOSTED_LOGS_ENDPOINT_URL\"), \"http://nginx.gateway.svc:3100/loki/api/v1/push\")\n}\n\n/********************************************\n * Logs\n ********************************************/\nimport.file \"logs\" {\n\tfilename - = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/logs\"\n}\n\nlogs.annotations_scrape - \"kubernetes\" {\n\tannotation_prefix = \"logs.grafana.com\"\n\tforward_to = - [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.keep_labels \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.logs_receiver]\n}\n\n/********************************************\n + = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/logs\"\n}\n\nlogs.kubernetes_cluster_events + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\n\tforward_to + = [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.annotations_scrape \"kubernetes\" + {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\n\tforward_to + = [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.keep_labels \"kubernetes\" + {\n\tforward_to = [provider.self_hosted_stack.kubernetes.logs_receiver]\n}\n\n/********************************************\n * Metrics\n ********************************************/\nimport.file \"metrics\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/metrics\"\n}\n\nmetrics.podmonitors_scrape \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape - \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\n/*****************************************************************\n* + + \"/kubernetes/metrics\"\n}\n\nmetrics.annotations_scrape \"kubernetes\" {\n\tcluster + \ = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape + \"kubernetes\" {\n\tcluster = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval + = \"15s\"\n\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\n/*****************************************************************\n* Alloy Integrations\n*****************************************************************/\nremote.kubernetes.configmap \"integrations\" {\n\tnamespace = \"monitoring-system\"\n\tname = \"alloy-integrations\"\n}\n\n// Memcached Integrations\nimport.string \"memcached\" {\n\tcontent = remote.kubernetes.configmap.integrations.data[\"memcached.alloy\"]\n}\n\nmemcached.memcached_metrics_scrape - \"instance\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\n\tnamespace - = \"monitoring-system\"\n\tname = remote.kubernetes.configmap.integrations.data[\"MEMCACHED_K8S_SECRET_NAME\"]\n}\n\n// - // Redis Integrations\n// import.string \"redis\" {\n// \tcontent = remote.kubernetes.configmap.integrations.data[\"redis.alloy\"]\n// - }\n\n// redis.redis_exporter_metrics_scrape \"instance\" {\n// \tforward_to = - [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\n// \tnamespace = \"monitoring-system\"\n// - \tname = remote.kubernetes.configmap.integrations.data[\"REDIS_K8S_SECRET_NAME\"]\n// - }\n\n// // Mysql Integrations\n// import.string \"mysql\" {\n// \tcontent = remote.kubernetes.configmap.integrations.data[\"mysql.alloy\"]\n// - }\n\n// mysql.mysql_metrics_scrape \"instance\" {\n// \tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\n// - \tnamespace = \"monitoring-system\"\n// \tname = remote.kubernetes.configmap.integrations.data[\"MYSQL_K8S_SECRET_NAME\"]\n// - }\n" + \"instance\" {\n\tnamespace = \"monitoring-system\"\n\tname = remote.kubernetes.configmap.integrations.data[\"MEMCACHED_K8S_SECRET_NAME\"]\n\n\tforward_to + = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-72927d7k99 + name: alloy-config-fmt85fgk68 namespace: monitoring-system --- apiVersion: v1 @@ -522,28 +518,25 @@ apiVersion: v1 data: annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: Scrapes targets for logs based on kubernetes Pod annotations\n\n Annotations:\n - \ logs.grafana.com/ingest: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare + \ logs.grafana.com/scrape: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this - can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\t// arguments - for kubernetes discovery\n\targument \"namespaces\" {\n\t\tcomment = \"The namespaces - to look for targets in (default: [\\\"kube-system\\\"] is all namespaces)\"\n\t\toptional - = true\n\t}\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\targument + \"cluster\" { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix to use (default: logs.grafana.com)\"\n\t\tdefault = \"logs.grafana.com\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional = true\n\t\tcomment \ = \"The logic is used to transform the annotation argument into a valid label name by removing unsupported characters.\"\n\t\tdefault = replace(replace(replace(coalesce(argument.annotation_prefix.value, \"logs.grafana.com\"), \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\t// - find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\n\t\tnamespaces - {\n\t\t\tnames = coalesce(argument.namespaces.value, [])\n\t\t}\n\t}\n\n\t// filter - logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" + find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\t}\n\n\t// + filter logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" {\n\t\ttargets = discovery.kubernetes.annotation_logs.targets\n\n\t\t// allow pods to declare their logs to be ingested or not, the default is true\n\t\t// - \ i.e. logs.grafana.com/ingest: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + \ i.e. logs.grafana.com/scrape: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\n\t\t// allow pods to declare what tenant their logs should be written to, the following annotation @@ -553,6 +546,8 @@ data: + argument.tenant.value + \")$\"\n\t\t}\n\n\t\t// set the instance label as the name of the worker node the pod is on\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_node_name\"]\n\t\t\ttarget_label = \"instance\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\t// set the pod label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\t// @@ -617,12 +612,11 @@ data: ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\toptional - = true\n\t}\n\n\targument \"cluster\" {\n\t\toptional = true\n\t}\n\n\tloki.source.kubernetes_events - \"cluster_events\" {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format + = true\n\t}\n\n\targument \"cluster\" { }\n\n\tloki.source.kubernetes_events \"cluster_events\" + {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format = \"logfmt\"\n\t\tforward_to = [loki.process.logs_service.receiver]\n\t}\n\n\tloki.process \"logs_service\" {\n\t\tstage.static_labels {\n\t\t\tvalues = {\n\t\t\t\tcluster - = coalesce(argument.cluster.value, \"k3d\"),\n\t\t\t}\n\t\t}\n\t\tforward_to = - argument.forward_to.value\n\t}\n}\n" + = argument.cluster.value,\n\t\t\t}\n\t\t}\n\t\tforward_to = argument.forward_to.value\n\t}\n}\n" keep-labels.alloy: "/*\nModule Components: keep_labels\nDescription: Pre-defined set of labels to keep, this stage should always be in-place as the previous relabeing\n \ stages make every pod label and annotation a label in the pipeline, @@ -644,46 +638,392 @@ data: = argument.keep_labels.value\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* EXPORTS\n\t*****************************************************************/\n\texport \"receiver\" {\n\t\tvalue = loki.process.keep_labels.receiver\n\t}\n}\n" + rules-to-loki.alloy: "/*\nModule Components: rules_to_loki\nDescription: Auto discovers + PrometheusRule Kubernetes resources and loads them into a Loki instance.\n*/\n\ndeclare + \"rules_to_loki\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + fake)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Loki\n\t********************************************/\n\tloki.rules.kubernetes + \"rules_to_loki\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:3100\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_loki= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\trule_selector {\n\t\t\tmatch_labels = {\n\t\t\t\tauto_rules_to_loki + = \"true\",\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k namespace: monitoring-system --- apiVersion: v1 data: + annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: + Scrapes targets for metrics based on kubernetes annotations\n\nNote: Every argument + except for \"forward_to\" is optional, and does have a defined default value. + \ However, the values for these\n arguments are not defined using the default + = \" ... \" argument syntax, but rather using the coalesce(argument.value, \" + ... \").\n This is because if the argument passed in from another consuming + module is set to null, the default = \" ... \" syntax will\n does not override + the value passed in, where coalesce() will return the first non-null value.\n\nKubernetes + Annotation Auto-Scraping\n------------------------------------------------------------------------------------------------------------------------------------\nThis + module is meant to be used to automatically scrape targets based on a certain + role and set of annotations. This module can be consumed\nmultiple times with + different roles. The supported roles are:\n\n - pod\n - service\n - endpoints\n\nTypically, + if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality + you would use role=\"endpoints\", if\nmimicking the behavior of the PodMonitor + functionality you would use role=\"pod\". It is important to understand that + with endpoints,\nthe target is typically going to be a pod, and whatever annotations + that are set on the service will automatically be propagated to the\nendpoints. + \ This is why the role \"endpoints\" is used, because it will scrape the pod, + but also consider the service annotations. Using\nrole=\"endpoints\", which scrape + each endpoint associated to the service. If role=\"service\" is used, it will + only scrape the service, only\nhitting one of the endpoints associated to the + service.\n\nThis is where you must consider your scraping strategy, for example + if you scrape a service like \"kube-state-metrics\" using\nrole=\"endpoints\" + you should only have a single replica of the kube-state-metrics pod, if you have + multiple replicas, you should use\nrole=\"service\" or a separate non-annotation + job completely. Scraping a service instead of endpoints, is typically a rare + use case, but\nit is supported.\n\nThere are other considerations for using annotation + based scraping as well, which is metric relabeling rules that happen post scrape. + \ If\nyou have a target that you want to apply a bunch of relabelings to or a + very large metrics response payload, performance wise it will be\nbetter to have + a separate job for that target, rather than using use annotations. As every targert + will go through the ssame relabeling.\nTypical deployment strategies/options would + be:\n\nOption #1 (recommended):\n - Annotation Scraping for role=\"endpoints\"\n + \ - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, + etc.) or large metric payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, + kube-apiserver, kube-scheduler, etc.)\n\nOption #2:\n - Annotation Scraping for + role=\"pod\"\n - Annotation Scraping for role=\"service\" (i.e. kube-state-metrics, + node-exporter, etc.)\n - Separate Jobs for specific use cases or large metric + payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, + kube-scheduler, etc.)\n\nAt no point should you use role=\"endpoints\" and role=\"pod\" + together, as this will result in duplicate targets being scraped, thus\ngenerating + duplicate metrics. If you want to scrape both the pod and the service, use Option + #2.\n\nEach port attached to an service/pod/endpoint is an eligible target, oftentimes + it will have multiple ports.\nThere may be instances when you want to scrape all + ports or some ports and not others. To support this\nthe following annotations + are available:\n\n metrics.grafana.com/scrape: true\n\nthe default scraping scheme + is http, this can be specified as a single value which would override, the schema + being used for all\nports attached to the target:\n\n metrics.grafana.com/scheme: + https\n\nthe default path to scrape is /metrics, this can be specified as a single + value which would override, the scrape path being used\nfor all ports attached + to the target:\n\n metrics.grafana.com/path: /metrics/some_path\n\nthe default + port to scrape is the target port, this can be specified as a single value which + would override the scrape port being\nused for all ports attached to the target, + note that even if aan target had multiple targets, the relabel_config targets + are\ndeduped before scraping:\n\n metrics.grafana.com/port: 8080\n\nthe default + interval to scrape is 1m, this can be specified as a single value which would + override, the scrape interval being used\nfor all ports attached to the target:\n\n + \ metrics.grafana.com/interval: 5m\n\nthe default timeout for scraping is 10s, + this can be specified as a single value which would override, the scrape interval + being\nused for all ports attached to the target:\n\n metrics.grafana.com/timeout: + 30s\n\nthe default job is namespace/{{ service name }} or namespace/{{ controller_name + }} depending on the role, there may be instances\nin which a different job name + is required because of a set of dashboards, rules, etc. to support this there + is a job annotation\nwhich will override the default value:\n\n metrics.grafana.com/job: + integrations/kubernetes/kube-state-metrics\n*/\n\ndeclare \"annotations_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment + \ = \"The tenant to filter metrics to. This does not have to be the tenantId, + this is the value to look for in the metrics.agent.grafana.com/tenant annotation, + and this can be a regex.\"\n\t\toptional = true\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + to use (default: metrics.grafana.com)\"\n\t\tdefault = \"metrics.grafana.com\"\n\t\toptional + = true\n\t}\n\n\targument \"role\" {\n\t\tcomment = \"The role to use when looking + for targets to scrape via annotations, can be: endpoints, service, pod (default: + endpoints)\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional + = true\n\t\tcomment = \"The logic is used to transform the annotation argument + into a valid label name by removing unsupported characters.\"\n\t\tdefault = + replace(replace(replace(coalesce(argument.annotation_prefix.value, \"metrics.grafana.com\"), + \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\targument \"__pod_role\" + {\n\t\tcomment = \"Most annotation targets service or pod that is all you want, + however if the role is endpoints you want the pod\"\n\t\toptional = true\n\t\tdefault + \ = replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"pod\")\n\t}\n\n\targument + \"__service_role\" {\n\t\tcomment = \"Most annotation targets service or pod + that is all you want, however if the role is endpoints you we also want to consider + service annotations\"\n\t\toptional = true\n\t\tdefault = replace(coalesce(argument.role.value, + \"endpoints\"), \"endpoints\", \"service\")\n\t}\n\n\targument \"scrape_port_named_metrics\" + {\n\t\tcomment = \"Whether or not to automatically scrape endpoints that have + a port with 'metrics' in the name\"\n\t\toptional = true\n\t\tdefault = false\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: (.+))\"\n\t\toptional + = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment = \"A regex of metrics + to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" + {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + Targets From Docker Discovery\n\t*****************************************************************/\n\tdiscovery.kubernetes + \"annotation_metrics\" {\n\t\trole = coalesce(argument.role.value, \"endpoints\")\n\n\t\tselectors + {\n\t\t\trole = coalesce(argument.role.value, \"endpoints\")\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t*****************************************************************/\n\t// + filter metrics by kubernetes annotations\n\tdiscovery.relabel \"annotation_metrics_filter\" + {\n\t\ttargets = discovery.kubernetes.annotation_metrics.targets\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Targets to Keep or Drop\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare their metrics scraped or not\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/scrape: false\n\t\t//\n\t\t// the label prometheus.io/service-monitor: + \"false\" is a common label for headless services, when performing endpoint\n\t\t// + service discovery, if there is both a load-balanced service and headless service, + this can result in duplicate\n\t\t// scrapes if the name of the service is attached + as a label. any targets with this label or annotation set should be dropped\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"false\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, + \"endpoints\") + \"_label_prometheus_io_service_monitor\",\n\t\t\t]\n\t\t\tseparator + = \";\"\n\t\t\t// only allow empty or true, otherwise defaults to false\n\t\t\tregex + \ = \"^(?:;*)?(true)(;|true)*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\t// add a __tmp_scrape_port_named_metrics from + the argument.scrape_port_named_metrics\n\t\trule {\n\t\t\treplacement = format(\"%t\", + argument.scrape_port_named_metrics.value)\n\t\t\ttarget_label = \"__tmp_scrape_port_named_metrics\"\n\t\t}\n\n\t\t// + only keep targets that have scrape: true or \"metrics\" in the port name if the + argument scrape_port_named_metrics\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__tmp_scrape\",\n\t\t\t\t\"__tmp_scrape_port_named_metrics\",\n\t\t\t\t// + endpoints is the role and most meta labels started with \"endpoints\", however + the port name is an exception and starts with \"endpoint\"\n\t\t\t\t\"__meta_kubernetes_\" + + replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"endpoint\") + + \"_port_name\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(true;.*|(|true);true;(.*metrics.*))$\"\n\t\t}\n\n\t\t// + only keep targets where the pod is running or the pod_phase is empty and is not + an init container. This will only exist for role=\"pod\" or\n\t\t// potentially + role=\"endpoints\", if it is a service the value is empty and thus allowed to + pass, if it is an endpoint but not associated to a\n\t\t// pod but rather a static + IP or hostname, that could be outside of kubernetes allow endpoints to declare + what tenant their metrics should be\n\t\t// written to\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex + \ = \"^(?i)(Running|)$\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__meta_kubernetes_pod_ready\"]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\t\t// + if the container is an init container, drop it\n\t\trule {\n\t\t\taction = + \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_container_init\"]\n\t\t\tregex + \ = \"^(true)$\"\n\t\t}\n\n\t\t// allow resources to declare their metrics + the tenant their metrics should be sent to,\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/tenant: primary\n\t\t//\n\t\t// Note: This does not necessarily + have to be the actual tenantId, it can be a friendly name as well that is simply + used\n\t\t// to determine if the metrics should be gathered for the current + tenant\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_tenant\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_tenant\",\n\t\t\t]\n\t\t\tregex + = \"^(\" + coalesce(argument.tenant.value, \".*\") + \")$\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Scrape Metadata i.e. path, port, interval etc.\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare the protocol to use when collecting metrics, the default + value is \"http\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/scheme: + http\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"http\"\n\t\t\ttarget_label + = \"__scheme__\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scheme\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scheme\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?(https?).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scheme__\"\n\t\t}\n\n\t\t// allow resources + to declare the port to use when collecting metrics, the default value is the discovered + port from\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/port: 9090\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__address__\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_port\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_port\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^:]+)(?::\\\\d+)?;(\\\\d+)$\"\n\t\t\treplacement + \ = \"$1:$2\"\n\t\t\ttarget_label = \"__address__\"\n\t\t}\n\n\t\t// allow resources + to declare their the path to use when collecting their metrics, the default value + is \"/metrics\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/path: + /metrics/foo\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_path\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_path\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__metrics_path__\"\n\t\t}\n\n\t\t// allow resources + to declare how often their metrics should be collected, the default value is 1m,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/interval: 5m\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_interval\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_interval\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\t// allow + resources to declare the timeout of the scrape request, the default value is 10s,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/timeout: 30s\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_timeout\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_timeout\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Common Labels\n\t\t****************************************************************************************************************/\n\t\t// + set a source label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = \"kubernetes\"\n\t\t\ttarget_label = \"source\"\n\t\t}\n\n\t\t// set the cluster + label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = argument.cluster.value\n\t\t\ttarget_label + = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label + \ = \"namespace\"\n\t\t}\n\n\t\t// set the target name label i.e. service name, + pod name, etc.\n\t\t// if the role is endpoints, the first valued field is used + which would be __meta_kubernetes_pod_name, if the pod name is empty\n\t\t// then + the endpoint name would be used\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + argument.__pod_role.value + \"_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = + \"$1\"\n\t\t\ttarget_label = argument.__pod_role.value\n\t\t}\n\n\t\t// set a + default job label to be the namespace/pod_controller_name or namespace/service_name\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t\targument.__pod_role.value,\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^;]+)(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// if the controller + is a ReplicaSet, drop the hash from the end of the ReplicaSet\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_type\",\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:ReplicaSet);([^;]+);([^;]+)-.+$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// allow resources to + declare their the job label value to use when collecting their metrics, the default + value is \"\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/job: + integrations/kubernetes/cadvisor\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_job\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_job\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// set the app name if + specified as metadata labels \"app:\" or \"app.kubernetes.io/name:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"app\"\n\t\t}\n\n\t\t// + set the app component if specified as metadata labels \"component:\" or \"app.kubernetes.io/component:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_component\",\n\t\t\t]\n\t\t\tregex + \ = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"component\"\n\t\t}\n\n\t\t// set the version if specified as metadata labels + \"version:\" or \"app.kubernetes.io/version:\" or \"app_version:\"\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_version\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"version\"\n\t\t}\n\n\t\t// + set a workload label if the resource is a pod\n\t\t// example: grafana-agent-68nv9 + becomes DaemonSet/grafana-agent\n\t\trule {\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_kind\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"(.+);(.+)\"\n\t\t\treplacement = \"$1/$2\"\n\t\t\ttarget_label + = \"workload\"\n\t\t}\n\t\t// remove the hash from the ReplicaSet\n\t\trule {\n\t\t\tsource_labels + = [\"workload\"]\n\t\t\tregex = \"(ReplicaSet/.+)-.+\"\n\t\t\ttarget_label + \ = \"workload\"\n\t\t}\n\t}\n\n\t// only keep http targets\n\tdiscovery.relabel + \"http_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"http\"\n\t\t}\n\t}\n\n\t// only keep https targets\n\tdiscovery.relabel + \"https_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"https\"\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Scrape Labels Targets\n\t*****************************************************************/\n\t// + scrape http only targtetsa\n\tprometheus.scrape \"http_annotations\" {\n\t\ttargets + = discovery.relabel.http_annotations.output\n\n\t\tjob_name = \"annotation-metrics-http\"\n\t\tscheme + \ = \"http\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t// + scrape https only targtets\n\tprometheus.scrape \"https_annotations\" {\n\t\ttargets + = discovery.relabel.https_annotations.output\n\n\t\tjob_name = \"annotation-metrics-https\"\n\t\tscheme + \ = \"https\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\tbearer_token_file + = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n\n\t\ttls_config {\n\t\t\tca_file + \ = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n\t\t\tinsecure_skip_verify + = false\n\t\t\tserver_name = \"kubernetes\"\n\t\t}\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"annotations\" {\n\t\tforward_to = argument.forward_to.value\n\t\t// keep only + metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(.+)\")\n\t\t}\n\n\t\t// drop metrics that match the drop_metrics regex\n\t\trule + {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex + \ = coalesce(argument.drop_metrics.value, \"\")\n\t\t}\n\t}\n}\n" podmonitors-scrape.alloy: "/*\nModule Components: podmonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.podmonitors\n*/\n\ndeclare \"podmonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape PodMonitors\n\t*****************************************************************/\n\tprometheus.operator.podmonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.podmonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\t// selector {\n\t\t// \tmatch_expression {\n\t\t// \t\tkey = \"team\"\n\t\t// \t\toperator = \"In\"\n\t\t// \t\tvalues - \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n}\n" + \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"podmonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set the + cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" + rules-to-mimir.alloy: "/*\nModule Components: rules_to_mimir\nDescription: Auto + discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance.\n*/\n\ndeclare + \"rules_to_mimir\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + anonymous)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Mimir\n\t********************************************/\n\tmimir.rules.kubernetes + \"rules_to_mimir\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:8080\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\t// rule_selector {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= + \"true\",\n\t\t// \t}\n\t\t// }\n\t}\n}\n" servicemonitors-scrape.alloy: "/*\nModule Components: servicemonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.servicemonitors\n*/\n\ndeclare \"servicemonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape ServiceMonitors\n\t*****************************************************************/\n\tprometheus.operator.servicemonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.servicemonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering - {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n" + {\n\t\t\tenabled = true\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"servicemonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set + the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f namespace: monitoring-system --- apiVersion: v1 @@ -693,19 +1033,21 @@ data: \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(ProfilessReceiver) where collected - logs should be forwarded to\"\n\t}\n\n\tdiscovery.kubernetes \"pyroscope_kubernetes\" - {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape config allows to define - annotations based scraping.\n\t//\n\t// For example the following annotations:\n\t//\n\t// - ```\n\t// profiles.grafana.com/memory.scrape: \"true\"\n\t// profiles.grafana.com/memory.port: - \"8080\"\n\t// profiles.grafana.com/cpu.scrape: \"true\"\n\t// profiles.grafana.com/cpu.port: - \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: \"true\"\n\t// profiles.grafana.com/goroutine.port: - \"8080\"\n\t// ```\n\t//\n\t// will scrape the `memory`, `cpu` and `goroutine` - profiles from the `8080` port of the pod.\n\t//\n\t// For more information see - https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel + logs should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\tdiscovery.kubernetes + \"pyroscope_kubernetes\" {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape + config allows to define annotations based scraping.\n\t//\n\t// For example the + following annotations:\n\t//\n\t// ```\n\t// profiles.grafana.com/memory.scrape: + \"true\"\n\t// profiles.grafana.com/memory.port: \"8080\"\n\t// profiles.grafana.com/cpu.scrape: + \"true\"\n\t// profiles.grafana.com/cpu.port: \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: + \"true\"\n\t// profiles.grafana.com/goroutine.port: \"8080\"\n\t// ```\n\t//\n\t// + will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port + of the pod.\n\t//\n\t// For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel \"kubernetes_pods\" {\n\t\ttargets = concat(discovery.kubernetes.pyroscope_kubernetes.targets)\n\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex \ = \"Pending|Succeeded|Failed|Completed\"\n\t\t}\n\n\t\trule {\n\t\t\taction - = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\trule + = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\trule @@ -924,7 +1266,7 @@ data: {\n\t\t\t\tenabled = true\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 namespace: monitoring-system --- apiVersion: v1 @@ -934,40 +1276,58 @@ data: Transformation\ndeclare \"process_and_transform\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"traces_forward_to\" {\n\t\tcomment = \"Must be a list(TracesReceiver) where - collected traces should be forwarded to\"\n\t}\n\n\targument \"logs_forward_to\" - {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be - forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment = \"Must - be a list(MetricsReceiver) where collected metrics should be forwarded to\"\n\t}\n\n\targument - \"cluster\" {\n\t\toptional = true\n\t\tdefault = \"k3d-k3s-codelab\"\n\t}\n\n\targument - \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4318\"\n\t}\n\n\targument - \"otlp_grpc_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* + collected traces should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\targument + \"logs_forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected + logs should be forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment + = \"Must be a list(MetricsReceiver) where collected metrics should be forwarded + to\"\n\t}\n\n\targument \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault + \ = \"0.0.0.0:4318\"\n\t}\n\n\targument \"otlp_grpc_endpoint\" {\n\t\toptional + = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* Jaeger for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.jaeger \"default\" {\n\t\tprotocols {\n\t\t\tgrpc {\n\t\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t\t}\n\n\t\t\tthrift_http {\n\t\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t\t}\n\n\t\t\tthrift_binary {\n\t\t\t\tendpoint = \"0.0.0.0:6832\"\n\t\t\t}\n\n\t\t\tthrift_compact {\n\t\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [otelcol.processor.resourcedetection.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + {\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* Otelcol for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.otlp - \"default\" {\n\t\tgrpc {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp - {\n\t\t\tendpoint = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics - = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection - \"default\" {\n\t\tdetectors = [\"env\"]\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces - = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes + \"default\" {\n\t\tdebug_metrics {\n\t\t\tdisable_high_cardinality_metrics = true\n\t\t}\n\n\t\tgrpc + {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp {\n\t\t\tendpoint + = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.resourcedetection.default.input]\n\t\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection + \"default\" {\n\t\tdetectors = [\"env\", \"system\"]\n\n\t\tsystem {\n\t\t\thostname_sources + = [\"os\"]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\n\t\t\tlogs + \ = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform + \"add_metric_datapoint_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements + {\n\t\t\tcontext = \"datapoint\"\n\t\t\tstatements = [\n\t\t\t\t\"set(attributes[\\\"deployment.environment\\\"], + resource.attributes[\\\"deployment.environment\\\"])\",\n\t\t\t\t\"set(attributes[\\\"service.version\\\"], + resource.attributes[\\\"service.version\\\"])\",\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes \"default\" {\n\t\textract {\n\t\t\tmetadata = [\n\t\t\t\t\"k8s.namespace.name\",\n\t\t\t\t\"k8s.pod.name\",\n\t\t\t\t\"k8s.deployment.name\",\n\t\t\t\t\"k8s.statefulset.name\",\n\t\t\t\t\"k8s.daemonset.name\",\n\t\t\t\t\"k8s.cronjob.name\",\n\t\t\t\t\"k8s.job.name\",\n\t\t\t\t\"k8s.node.name\",\n\t\t\t\t\"k8s.pod.uid\",\n\t\t\t\t\"k8s.pod.start_time\",\n\t\t\t]\n\t\t}\n\n\t\tpod_association {\n\t\t\tsource {\n\t\t\t\tfrom = \"connection\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t\ttraces - = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform - \"add_resource_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tlog_statements + {\n\t\t\tmetrics = [otelcol.processor.transform.default.input]\n\t\t\tlogs = + [otelcol.processor.transform.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.transform.default.input,\n\t\t\t\totelcol.connector.host_info.default.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.connector.host_info + \"default\" {\n\t\thost_identifiers = [\"k8s.node.name\"]\n\n\t\toutput {\n\t\t\tmetrics + = [otelcol.processor.batch.host_info_batch.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + \"host_info_batch\" {\n\t\toutput {\n\t\t\tmetrics = [otelcol.exporter.prometheus.host_info_metrics.input]\n\t\t}\n\t}\n\n\totelcol.exporter.prometheus + \"host_info_metrics\" {\n\t\tadd_metric_suffixes = false\n\t\tforward_to = + argument.metrics_forward_to.value\n\t}\n\n\totelcol.processor.transform \"default\" + {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements {\n\t\t\tcontext = + \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\tlog_statements {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])`,\n\t\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t\t`set(attributes[\"loki.resource.labels\"], \"pod, namespace, cluster, job\")`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements - {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], - \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter - \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.batch.default.input]\n\t\t\ttraces - = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements + {\n\t\t\tcontext = \"span\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces + \ = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter + \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\ttraces {\n\t\t\tspan = [\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/live\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] == \\\"/healthy\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/ready\\\"\",\n\t\t\t]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs + \ = [otelcol.processor.batch.default.input]\n\t\t\ttraces = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch \"default\" {\n\t\tsend_batch_size = 16384\n\t\tsend_batch_max_size = 0\n\t\ttimeout \ = \"5s\"\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\t\tlogs \ = [otelcol.processor.memory_limiter.default.input]\n\t\t\ttraces = [otelcol.processor.memory_limiter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.memory_limiter @@ -1003,7 +1363,7 @@ data: \"alloy_traces_input\" {\n\t\tvalue = otelcol.processor.batch.default.input\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 namespace: monitoring-system --- apiVersion: v1 @@ -2103,19 +2463,19 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-72927d7k99 + name: alloy-config-fmt85fgk68 name: config - configMap: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f name: modules-kubernetes-metrics - configMap: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k name: modules-kubernetes-logs - configMap: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 name: modules-kubernetes-traces - configMap: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 name: modules-kubernetes-profiles --- apiVersion: policy/v1 diff --git a/kubernetes/read-write-mode/logs/logs.alloy b/kubernetes/read-write-mode/logs/logs.alloy index 514a59e7..f7d59991 100644 --- a/kubernetes/read-write-mode/logs/logs.alloy +++ b/kubernetes/read-write-mode/logs/logs.alloy @@ -25,9 +25,16 @@ import.file "logs" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/logs" } +logs.kubernetes_cluster_events "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + + forward_to = [logs.keep_labels.kubernetes.receiver] +} + logs.annotations_scrape "kubernetes" { - annotation_prefix = "logs.grafana.com" - forward_to = [logs.keep_labels.kubernetes.receiver] + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + + forward_to = [logs.keep_labels.kubernetes.receiver] } logs.keep_labels "kubernetes" { @@ -41,11 +48,17 @@ import.file "metrics" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/metrics" } -metrics.podmonitors_scrape "kubernetes" { +metrics.annotations_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } metrics.servicemonitors_scrape "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } @@ -63,32 +76,8 @@ import.string "memcached" { } memcached.memcached_metrics_scrape "instance" { - forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] - namespace = "monitoring-system" name = remote.kubernetes.configmap.integrations.data["MEMCACHED_K8S_SECRET_NAME"] -} - -// // Redis Integrations -// import.string "redis" { -// content = remote.kubernetes.configmap.integrations.data["redis.alloy"] -// } - -// redis.redis_exporter_metrics_scrape "instance" { -// forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] - -// namespace = "monitoring-system" -// name = remote.kubernetes.configmap.integrations.data["REDIS_K8S_SECRET_NAME"] -// } -// // Mysql Integrations -// import.string "mysql" { -// content = remote.kubernetes.configmap.integrations.data["mysql.alloy"] -// } - -// mysql.mysql_metrics_scrape "instance" { -// forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] - -// namespace = "monitoring-system" -// name = remote.kubernetes.configmap.integrations.data["MYSQL_K8S_SECRET_NAME"] -// } + forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] +} diff --git a/kubernetes/read-write-mode/metrics/k8s-all-in-one.yaml b/kubernetes/read-write-mode/metrics/k8s-all-in-one.yaml index 6ab48b9a..803306cb 100644 --- a/kubernetes/read-write-mode/metrics/k8s-all-in-one.yaml +++ b/kubernetes/read-write-mode/metrics/k8s-all-in-one.yaml @@ -154,43 +154,38 @@ data: = coalesce(env(\"SELF_HOSTED_METRICS_ENDPOINT_URL\"), \"http://nginx.gateway.svc:8080/api/v1/push\")\n}\n\n/********************************************\n * Metrics\n ********************************************/\nimport.file \"metrics\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") - + \"/kubernetes/metrics\"\n}\n\nmetrics.podmonitors_scrape \"kubernetes\" {\n\tforward_to - = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\nmetrics.servicemonitors_scrape - \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n\n// - metrics.annotations_scrape \"kubernetes\" {\n// \tlabel_prefix = \"metrics.grafana.com\"\n// - \tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n// - \tscrape_interval = \"15s\"\n// }\n" + + \"/kubernetes/metrics\"\n}\n\nmetrics.annotations_scrape \"kubernetes\" {\n\tforward_to + = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n\n\tcluster = + coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\tscrape_interval = \"15s\"\n}\n\nmetrics.servicemonitors_scrape + \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-55dd47md5f + name: alloy-config-dkdkkgdd64 namespace: monitoring-system --- apiVersion: v1 data: annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: Scrapes targets for logs based on kubernetes Pod annotations\n\n Annotations:\n - \ logs.grafana.com/ingest: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare + \ logs.grafana.com/scrape: true\n logs.grafana.com/tenant: \"primary\"\n*/\n\ndeclare \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this - can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\t// arguments - for kubernetes discovery\n\targument \"namespaces\" {\n\t\tcomment = \"The namespaces - to look for targets in (default: [\\\"kube-system\\\"] is all namespaces)\"\n\t\toptional - = true\n\t}\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + can be a regex.\"\n\t\toptional = true\n\t\tdefault = \".*\"\n\t}\n\n\targument + \"cluster\" { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix to use (default: logs.grafana.com)\"\n\t\tdefault = \"logs.grafana.com\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional = true\n\t\tcomment \ = \"The logic is used to transform the annotation argument into a valid label name by removing unsupported characters.\"\n\t\tdefault = replace(replace(replace(coalesce(argument.annotation_prefix.value, \"logs.grafana.com\"), \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\t// - find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\n\t\tnamespaces - {\n\t\t\tnames = coalesce(argument.namespaces.value, [])\n\t\t}\n\t}\n\n\t// filter - logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" + find all pods\n\tdiscovery.kubernetes \"annotation_logs\" {\n\t\trole = \"pod\"\n\t}\n\n\t// + filter logs by kubernetes annotations\n\tdiscovery.relabel \"annotation_logs_filter\" {\n\t\ttargets = discovery.kubernetes.annotation_logs.targets\n\n\t\t// allow pods to declare their logs to be ingested or not, the default is true\n\t\t// - \ i.e. logs.grafana.com/ingest: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + \ i.e. logs.grafana.com/scrape: false\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\n\t\t// allow pods to declare what tenant their logs should be written to, the following annotation @@ -200,6 +195,8 @@ data: + argument.tenant.value + \")$\"\n\t\t}\n\n\t\t// set the instance label as the name of the worker node the pod is on\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_node_name\"]\n\t\t\ttarget_label = \"instance\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\t// set the pod label\n\t\trule {\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\t// @@ -264,12 +261,11 @@ data: ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\toptional - = true\n\t}\n\n\targument \"cluster\" {\n\t\toptional = true\n\t}\n\n\tloki.source.kubernetes_events - \"cluster_events\" {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format + = true\n\t}\n\n\targument \"cluster\" { }\n\n\tloki.source.kubernetes_events \"cluster_events\" + {\n\t\tjob_name = coalesce(argument.job_label.value, \"integrations/kubernetes/eventhandler\")\n\t\tlog_format = \"logfmt\"\n\t\tforward_to = [loki.process.logs_service.receiver]\n\t}\n\n\tloki.process \"logs_service\" {\n\t\tstage.static_labels {\n\t\t\tvalues = {\n\t\t\t\tcluster - = coalesce(argument.cluster.value, \"k3d\"),\n\t\t\t}\n\t\t}\n\t\tforward_to = - argument.forward_to.value\n\t}\n}\n" + = argument.cluster.value,\n\t\t\t}\n\t\t}\n\t\tforward_to = argument.forward_to.value\n\t}\n}\n" keep-labels.alloy: "/*\nModule Components: keep_labels\nDescription: Pre-defined set of labels to keep, this stage should always be in-place as the previous relabeing\n \ stages make every pod label and annotation a label in the pipeline, @@ -291,46 +287,392 @@ data: = argument.keep_labels.value\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* EXPORTS\n\t*****************************************************************/\n\texport \"receiver\" {\n\t\tvalue = loki.process.keep_labels.receiver\n\t}\n}\n" + rules-to-loki.alloy: "/*\nModule Components: rules_to_loki\nDescription: Auto discovers + PrometheusRule Kubernetes resources and loads them into a Loki instance.\n*/\n\ndeclare + \"rules_to_loki\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Loki ruler. (default: http://nginx.gateway.svc:3100)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + fake)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Loki\n\t********************************************/\n\tloki.rules.kubernetes + \"rules_to_loki\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:3100\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_loki= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\trule_selector {\n\t\t\tmatch_labels = {\n\t\t\t\tauto_rules_to_loki + = \"true\",\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k namespace: monitoring-system --- apiVersion: v1 data: + annotations-scrape.alloy: "/*\nModule Components: annotations_scrape\nDescription: + Scrapes targets for metrics based on kubernetes annotations\n\nNote: Every argument + except for \"forward_to\" is optional, and does have a defined default value. + \ However, the values for these\n arguments are not defined using the default + = \" ... \" argument syntax, but rather using the coalesce(argument.value, \" + ... \").\n This is because if the argument passed in from another consuming + module is set to null, the default = \" ... \" syntax will\n does not override + the value passed in, where coalesce() will return the first non-null value.\n\nKubernetes + Annotation Auto-Scraping\n------------------------------------------------------------------------------------------------------------------------------------\nThis + module is meant to be used to automatically scrape targets based on a certain + role and set of annotations. This module can be consumed\nmultiple times with + different roles. The supported roles are:\n\n - pod\n - service\n - endpoints\n\nTypically, + if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality + you would use role=\"endpoints\", if\nmimicking the behavior of the PodMonitor + functionality you would use role=\"pod\". It is important to understand that + with endpoints,\nthe target is typically going to be a pod, and whatever annotations + that are set on the service will automatically be propagated to the\nendpoints. + \ This is why the role \"endpoints\" is used, because it will scrape the pod, + but also consider the service annotations. Using\nrole=\"endpoints\", which scrape + each endpoint associated to the service. If role=\"service\" is used, it will + only scrape the service, only\nhitting one of the endpoints associated to the + service.\n\nThis is where you must consider your scraping strategy, for example + if you scrape a service like \"kube-state-metrics\" using\nrole=\"endpoints\" + you should only have a single replica of the kube-state-metrics pod, if you have + multiple replicas, you should use\nrole=\"service\" or a separate non-annotation + job completely. Scraping a service instead of endpoints, is typically a rare + use case, but\nit is supported.\n\nThere are other considerations for using annotation + based scraping as well, which is metric relabeling rules that happen post scrape. + \ If\nyou have a target that you want to apply a bunch of relabelings to or a + very large metrics response payload, performance wise it will be\nbetter to have + a separate job for that target, rather than using use annotations. As every targert + will go through the ssame relabeling.\nTypical deployment strategies/options would + be:\n\nOption #1 (recommended):\n - Annotation Scraping for role=\"endpoints\"\n + \ - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, + etc.) or large metric payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, + kube-apiserver, kube-scheduler, etc.)\n\nOption #2:\n - Annotation Scraping for + role=\"pod\"\n - Annotation Scraping for role=\"service\" (i.e. kube-state-metrics, + node-exporter, etc.)\n - Separate Jobs for specific use cases or large metric + payloads\n - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, + kube-scheduler, etc.)\n\nAt no point should you use role=\"endpoints\" and role=\"pod\" + together, as this will result in duplicate targets being scraped, thus\ngenerating + duplicate metrics. If you want to scrape both the pod and the service, use Option + #2.\n\nEach port attached to an service/pod/endpoint is an eligible target, oftentimes + it will have multiple ports.\nThere may be instances when you want to scrape all + ports or some ports and not others. To support this\nthe following annotations + are available:\n\n metrics.grafana.com/scrape: true\n\nthe default scraping scheme + is http, this can be specified as a single value which would override, the schema + being used for all\nports attached to the target:\n\n metrics.grafana.com/scheme: + https\n\nthe default path to scrape is /metrics, this can be specified as a single + value which would override, the scrape path being used\nfor all ports attached + to the target:\n\n metrics.grafana.com/path: /metrics/some_path\n\nthe default + port to scrape is the target port, this can be specified as a single value which + would override the scrape port being\nused for all ports attached to the target, + note that even if aan target had multiple targets, the relabel_config targets + are\ndeduped before scraping:\n\n metrics.grafana.com/port: 8080\n\nthe default + interval to scrape is 1m, this can be specified as a single value which would + override, the scrape interval being used\nfor all ports attached to the target:\n\n + \ metrics.grafana.com/interval: 5m\n\nthe default timeout for scraping is 10s, + this can be specified as a single value which would override, the scrape interval + being\nused for all ports attached to the target:\n\n metrics.grafana.com/timeout: + 30s\n\nthe default job is namespace/{{ service name }} or namespace/{{ controller_name + }} depending on the role, there may be instances\nin which a different job name + is required because of a set of dashboards, rules, etc. to support this there + is a job annotation\nwhich will override the default value:\n\n metrics.grafana.com/job: + integrations/kubernetes/kube-state-metrics\n*/\n\ndeclare \"annotations_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"tenant\" {\n\t\tcomment + \ = \"The tenant to filter metrics to. This does not have to be the tenantId, + this is the value to look for in the metrics.agent.grafana.com/tenant annotation, + and this can be a regex.\"\n\t\toptional = true\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"annotation_prefix\" {\n\t\tcomment = \"The annotation_prefix + to use (default: metrics.grafana.com)\"\n\t\tdefault = \"metrics.grafana.com\"\n\t\toptional + = true\n\t}\n\n\targument \"role\" {\n\t\tcomment = \"The role to use when looking + for targets to scrape via annotations, can be: endpoints, service, pod (default: + endpoints)\"\n\t\toptional = true\n\t}\n\n\targument \"__sd_annotation\" {\n\t\toptional + = true\n\t\tcomment = \"The logic is used to transform the annotation argument + into a valid label name by removing unsupported characters.\"\n\t\tdefault = + replace(replace(replace(coalesce(argument.annotation_prefix.value, \"metrics.grafana.com\"), + \".\", \"_\"), \"/\", \"_\"), \"-\", \"_\")\n\t}\n\n\targument \"__pod_role\" + {\n\t\tcomment = \"Most annotation targets service or pod that is all you want, + however if the role is endpoints you want the pod\"\n\t\toptional = true\n\t\tdefault + \ = replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"pod\")\n\t}\n\n\targument + \"__service_role\" {\n\t\tcomment = \"Most annotation targets service or pod + that is all you want, however if the role is endpoints you we also want to consider + service annotations\"\n\t\toptional = true\n\t\tdefault = replace(coalesce(argument.role.value, + \"endpoints\"), \"endpoints\", \"service\")\n\t}\n\n\targument \"scrape_port_named_metrics\" + {\n\t\tcomment = \"Whether or not to automatically scrape endpoints that have + a port with 'metrics' in the name\"\n\t\toptional = true\n\t\tdefault = false\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: (.+))\"\n\t\toptional + = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment = \"A regex of metrics + to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" + {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + Targets From Docker Discovery\n\t*****************************************************************/\n\tdiscovery.kubernetes + \"annotation_metrics\" {\n\t\trole = coalesce(argument.role.value, \"endpoints\")\n\n\t\tselectors + {\n\t\t\trole = coalesce(argument.role.value, \"endpoints\")\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t*****************************************************************/\n\t// + filter metrics by kubernetes annotations\n\tdiscovery.relabel \"annotation_metrics_filter\" + {\n\t\ttargets = discovery.kubernetes.annotation_metrics.targets\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Targets to Keep or Drop\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare their metrics scraped or not\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/scrape: false\n\t\t//\n\t\t// the label prometheus.io/service-monitor: + \"false\" is a common label for headless services, when performing endpoint\n\t\t// + service discovery, if there is both a load-balanced service and headless service, + this can result in duplicate\n\t\t// scrapes if the name of the service is attached + as a label. any targets with this label or annotation set should be dropped\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"false\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scrape\",\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, + \"endpoints\") + \"_label_prometheus_io_service_monitor\",\n\t\t\t]\n\t\t\tseparator + = \";\"\n\t\t\t// only allow empty or true, otherwise defaults to false\n\t\t\tregex + \ = \"^(?:;*)?(true)(;|true)*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"__tmp_scrape\"\n\t\t}\n\n\t\t// add a __tmp_scrape_port_named_metrics from + the argument.scrape_port_named_metrics\n\t\trule {\n\t\t\treplacement = format(\"%t\", + argument.scrape_port_named_metrics.value)\n\t\t\ttarget_label = \"__tmp_scrape_port_named_metrics\"\n\t\t}\n\n\t\t// + only keep targets that have scrape: true or \"metrics\" in the port name if the + argument scrape_port_named_metrics\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__tmp_scrape\",\n\t\t\t\t\"__tmp_scrape_port_named_metrics\",\n\t\t\t\t// + endpoints is the role and most meta labels started with \"endpoints\", however + the port name is an exception and starts with \"endpoint\"\n\t\t\t\t\"__meta_kubernetes_\" + + replace(coalesce(argument.role.value, \"endpoints\"), \"endpoints\", \"endpoint\") + + \"_port_name\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(true;.*|(|true);true;(.*metrics.*))$\"\n\t\t}\n\n\t\t// + only keep targets where the pod is running or the pod_phase is empty and is not + an init container. This will only exist for role=\"pod\" or\n\t\t// potentially + role=\"endpoints\", if it is a service the value is empty and thus allowed to + pass, if it is an endpoint but not associated to a\n\t\t// pod but rather a static + IP or hostname, that could be outside of kubernetes allow endpoints to declare + what tenant their metrics should be\n\t\t// written to\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex + \ = \"^(?i)(Running|)$\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__meta_kubernetes_pod_ready\"]\n\t\t\tregex = \"^(true|)$\"\n\t\t}\n\t\t// + if the container is an init container, drop it\n\t\trule {\n\t\t\taction = + \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_container_init\"]\n\t\t\tregex + \ = \"^(true)$\"\n\t\t}\n\n\t\t// allow resources to declare their metrics + the tenant their metrics should be sent to,\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/tenant: primary\n\t\t//\n\t\t// Note: This does not necessarily + have to be the actual tenantId, it can be a friendly name as well that is simply + used\n\t\t// to determine if the metrics should be gathered for the current + tenant\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_tenant\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_tenant\",\n\t\t\t]\n\t\t\tregex + = \"^(\" + coalesce(argument.tenant.value, \".*\") + \")$\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Scrape Metadata i.e. path, port, interval etc.\n\t\t****************************************************************************************************************/\n\t\t// + allow resources to declare the protocol to use when collecting metrics, the default + value is \"http\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/scheme: + http\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = \"http\"\n\t\t\ttarget_label + = \"__scheme__\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_scheme\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_scheme\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?(https?).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scheme__\"\n\t\t}\n\n\t\t// allow resources + to declare the port to use when collecting metrics, the default value is the discovered + port from\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/port: 9090\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__address__\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_port\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_port\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^:]+)(?::\\\\d+)?;(\\\\d+)$\"\n\t\t\treplacement + \ = \"$1:$2\"\n\t\t\ttarget_label = \"__address__\"\n\t\t}\n\n\t\t// allow resources + to declare their the path to use when collecting their metrics, the default value + is \"/metrics\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/path: + /metrics/foo\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_path\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_path\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__metrics_path__\"\n\t\t}\n\n\t\t// allow resources + to declare how often their metrics should be collected, the default value is 1m,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/interval: 5m\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_interval\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_interval\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_interval__\"\n\t\t}\n\n\t\t// allow + resources to declare the timeout of the scrape request, the default value is 10s,\n\t\t// + the following duration formats are supported (s|m|ms|h|d):\n\t\t// Example Annotation:\n\t\t// + \ metrics.grafana.com/timeout: 30s\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_annotation_\" + argument.__sd_annotation.value + + \"_timeout\",\n\t\t\t\t\"__meta_kubernetes_\" + argument.__service_role.value + + \"_annotation_\" + argument.__sd_annotation.value + \"_timeout\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?(\\\\d+(s|m|ms|h|d)).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"__scrape_timeout__\"\n\t\t}\n\n\t\t/****************************************************************************************************************\n\t\t* + Handle Setting Common Labels\n\t\t****************************************************************************************************************/\n\t\t// + set a source label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = \"kubernetes\"\n\t\t\ttarget_label = \"source\"\n\t\t}\n\n\t\t// set the cluster + label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement = argument.cluster.value\n\t\t\ttarget_label + = \"cluster\"\n\t\t}\n\n\t\t// set the namespace label\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label + \ = \"namespace\"\n\t\t}\n\n\t\t// set the target name label i.e. service name, + pod name, etc.\n\t\t// if the role is endpoints, the first valued field is used + which would be __meta_kubernetes_pod_name, if the pod name is empty\n\t\t// then + the endpoint name would be used\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + argument.__pod_role.value + \"_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = + \"$1\"\n\t\t\ttarget_label = argument.__pod_role.value\n\t\t}\n\n\t\t// set a + default job label to be the namespace/pod_controller_name or namespace/service_name\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t\targument.__pod_role.value,\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^([^;]+)(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// if the controller + is a ReplicaSet, drop the hash from the end of the ReplicaSet\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_type\",\n\t\t\t\t\"__meta_kubernetes_namespace\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"^(?:ReplicaSet);([^;]+);([^;]+)-.+$\"\n\t\t\treplacement + \ = \"$1/$2\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// allow resources to + declare their the job label value to use when collecting their metrics, the default + value is \"\",\n\t\t// Example Annotation:\n\t\t// metrics.grafana.com/job: + integrations/kubernetes/cadvisor\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels + = [\n\t\t\t\t\"__meta_kubernetes_\" + coalesce(argument.role.value, \"endpoints\") + + \"_annotation_\" + argument.__sd_annotation.value + \"_job\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_annotation_\" + argument.__sd_annotation.value + + \"_job\",\n\t\t\t]\n\t\t\tseparator = \";\"\n\t\t\tregex = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement + \ = \"$1\"\n\t\t\ttarget_label = \"job\"\n\t\t}\n\n\t\t// set the app name if + specified as metadata labels \"app:\" or \"app.kubernetes.io/name:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_name\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_k8s_app\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"app\"\n\t\t}\n\n\t\t// + set the app component if specified as metadata labels \"component:\" or \"app.kubernetes.io/component:\"\n\t\trule + {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_component\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_component\",\n\t\t\t]\n\t\t\tregex + \ = \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label + = \"component\"\n\t\t}\n\n\t\t// set the version if specified as metadata labels + \"version:\" or \"app.kubernetes.io/version:\" or \"app_version:\"\n\t\trule {\n\t\t\taction + \ = \"replace\"\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + coalesce(argument.role.value, \"endpoints\") + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__pod_role.value + \"_label_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_app_kubernetes_io_version\",\n\t\t\t\t\"__meta_kubernetes_\" + + argument.__service_role.value + \"_label_version\",\n\t\t\t]\n\t\t\tregex = + \"^(?:;*)?([^;]+).*$\"\n\t\t\treplacement = \"$1\"\n\t\t\ttarget_label = \"version\"\n\t\t}\n\n\t\t// + set a workload label if the resource is a pod\n\t\t// example: grafana-agent-68nv9 + becomes DaemonSet/grafana-agent\n\t\trule {\n\t\t\tsource_labels = [\n\t\t\t\t\"__meta_kubernetes_pod_controller_kind\",\n\t\t\t\t\"__meta_kubernetes_pod_controller_name\",\n\t\t\t]\n\t\t\tseparator + \ = \";\"\n\t\t\tregex = \"(.+);(.+)\"\n\t\t\treplacement = \"$1/$2\"\n\t\t\ttarget_label + = \"workload\"\n\t\t}\n\t\t// remove the hash from the ReplicaSet\n\t\trule {\n\t\t\tsource_labels + = [\"workload\"]\n\t\t\tregex = \"(ReplicaSet/.+)-.+\"\n\t\t\ttarget_label + \ = \"workload\"\n\t\t}\n\t}\n\n\t// only keep http targets\n\tdiscovery.relabel + \"http_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"http\"\n\t\t}\n\t}\n\n\t// only keep https targets\n\tdiscovery.relabel + \"https_annotations\" {\n\t\ttargets = discovery.relabel.annotation_metrics_filter.output\n\n\t\trule + {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels = [\"__scheme__\"]\n\t\t\tregex + \ = \"https\"\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Scrape Labels Targets\n\t*****************************************************************/\n\t// + scrape http only targtetsa\n\tprometheus.scrape \"http_annotations\" {\n\t\ttargets + = discovery.relabel.http_annotations.output\n\n\t\tjob_name = \"annotation-metrics-http\"\n\t\tscheme + \ = \"http\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t// + scrape https only targtets\n\tprometheus.scrape \"https_annotations\" {\n\t\ttargets + = discovery.relabel.https_annotations.output\n\n\t\tjob_name = \"annotation-metrics-https\"\n\t\tscheme + \ = \"https\"\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\tbearer_token_file + = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n\n\t\ttls_config {\n\t\t\tca_file + \ = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n\t\t\tinsecure_skip_verify + = false\n\t\t\tserver_name = \"kubernetes\"\n\t\t}\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tclustering {\n\t\t\tenabled + = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.annotations.receiver]\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"annotations\" {\n\t\tforward_to = argument.forward_to.value\n\t\t// keep only + metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction = \"keep\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(.+)\")\n\t\t}\n\n\t\t// drop metrics that match the drop_metrics regex\n\t\trule + {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex + \ = coalesce(argument.drop_metrics.value, \"\")\n\t\t}\n\t}\n}\n" podmonitors-scrape.alloy: "/*\nModule Components: podmonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.podmonitors\n*/\n\ndeclare \"podmonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape PodMonitors\n\t*****************************************************************/\n\tprometheus.operator.podmonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.podmonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\t// selector {\n\t\t// \tmatch_expression {\n\t\t// \t\tkey = \"team\"\n\t\t// \t\toperator = \"In\"\n\t\t// \t\tvalues - \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n}\n" + \ = [\"team-infra\"]\n\t\t// \t}\n\t\t// }\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"podmonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set the + cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" + rules-to-mimir.alloy: "/*\nModule Components: rules_to_mimir\nDescription: Auto + discovers PrometheusRule Kubernetes resources and loads them into a Mimir instance.\n*/\n\ndeclare + \"rules_to_mimir\" {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"address\" {\n\t\tcomment = \"URL of the Mimir ruler. (default: http://nginx.gateway.svc:8080)\"\n\t\toptional + = true\n\t}\n\n\targument \"tenant\" {\n\t\tcomment = \"Mimir tenant ID. (default: + anonymous)\"\n\t\toptional = true\n\t}\n\n\t/********************************************\n\t* + Kubernetes Prometheus Rules To Mimir\n\t********************************************/\n\tmimir.rules.kubernetes + \"rules_to_mimir\" {\n\t\taddress = coalesce(argument.address.value, \"http://nginx.gateway.svc:8080\")\n\t\ttenant_id + = coalesce(argument.tenant.value, \"anonymous\")\n\n\t\t// rule_namespace_selector + {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= \"true\",\n\t\t// + \t}\n\t\t// }\n\n\t\t// rule_selector {\n\t\t// \tmatch_labels = {\n\t\t// \t\tauto_rules_to_mimir= + \"true\",\n\t\t// \t}\n\t\t// }\n\t}\n}\n" servicemonitors-scrape.alloy: "/*\nModule Components: servicemonitors_scrape\nDescription: Scrapes targets for metrics based on prometheus.operator.servicemonitors\n*/\n\ndeclare \"servicemonitors_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected - metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"scrape_interval\" - {\n\t\tcomment = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional - = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before - a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* + metrics should be forwarded to\"\n\t\toptional = false\n\t}\n\n\targument \"cluster\" + { }\n\n\targument \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep + (default: (.+))\"\n\t\toptional = true\n\t}\n\n\targument \"drop_metrics\" {\n\t\tcomment + \ = \"A regex of metrics to drop (default: \\\"\\\")\"\n\t\toptional = true\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\t/*****************************************************************\n\t* Kubernetes Auto Scrape ServiceMonitors\n\t*****************************************************************/\n\tprometheus.operator.servicemonitors - \"scrape\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\tscrape {\n\t\t\tdefault_scrape_interval - = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout + \"scrape\" {\n\t\tforward_to = [prometheus.relabel.servicemonitors.receiver]\n\n\t\tscrape + {\n\t\t\tdefault_scrape_interval = coalesce(argument.scrape_interval.value, \"60s\")\n\t\t\tdefault_scrape_timeout \ = coalesce(argument.scrape_timeout.value, \"10s\")\n\t\t}\n\n\t\tclustering - {\n\t\t\tenabled = true\n\t\t}\n\t}\n}\n" + {\n\t\t\tenabled = true\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t*****************************************************************/\n\t// + perform generic relabeling using keep_metrics and drop_metrics\n\tprometheus.relabel + \"servicemonitors\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// set + the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\taction + \ = \"keep\"\n\t\t\tsource_labels = [\"__name__\"]\n\t\t\tregex = + coalesce(argument.keep_metrics.value, \"(.+)\")\n\t\t}\n\n\t\t// drop metrics + that match the drop_metrics regex\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.drop_metrics.value, + \"\")\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f namespace: monitoring-system --- apiVersion: v1 @@ -340,19 +682,21 @@ data: \"annotations_scrape\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"forward_to\" {\n\t\tcomment = \"Must be a list(ProfilessReceiver) where collected - logs should be forwarded to\"\n\t}\n\n\tdiscovery.kubernetes \"pyroscope_kubernetes\" - {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape config allows to define - annotations based scraping.\n\t//\n\t// For example the following annotations:\n\t//\n\t// - ```\n\t// profiles.grafana.com/memory.scrape: \"true\"\n\t// profiles.grafana.com/memory.port: - \"8080\"\n\t// profiles.grafana.com/cpu.scrape: \"true\"\n\t// profiles.grafana.com/cpu.port: - \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: \"true\"\n\t// profiles.grafana.com/goroutine.port: - \"8080\"\n\t// ```\n\t//\n\t// will scrape the `memory`, `cpu` and `goroutine` - profiles from the `8080` port of the pod.\n\t//\n\t// For more information see - https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel + logs should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\tdiscovery.kubernetes + \"pyroscope_kubernetes\" {\n\t\trole = \"pod\"\n\t}\n\n\t// The default scrape + config allows to define annotations based scraping.\n\t//\n\t// For example the + following annotations:\n\t//\n\t// ```\n\t// profiles.grafana.com/memory.scrape: + \"true\"\n\t// profiles.grafana.com/memory.port: \"8080\"\n\t// profiles.grafana.com/cpu.scrape: + \"true\"\n\t// profiles.grafana.com/cpu.port: \"8080\"\n\t// profiles.grafana.com/goroutine.scrape: + \"true\"\n\t// profiles.grafana.com/goroutine.port: \"8080\"\n\t// ```\n\t//\n\t// + will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port + of the pod.\n\t//\n\t// For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\n\tdiscovery.relabel \"kubernetes_pods\" {\n\t\ttargets = concat(discovery.kubernetes.pyroscope_kubernetes.targets)\n\n\t\trule {\n\t\t\taction = \"drop\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_phase\"]\n\t\t\tregex \ = \"Pending|Succeeded|Failed|Completed\"\n\t\t}\n\n\t\trule {\n\t\t\taction - = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\trule + = \"labelmap\"\n\t\t\tregex = \"__meta_kubernetes_pod_label_(.+)\"\n\t\t}\n\n\t\t// + set the cluster label\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\treplacement + \ = argument.cluster.value\n\t\t\ttarget_label = \"cluster\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_namespace\"]\n\t\t\ttarget_label \ = \"namespace\"\n\t\t}\n\n\t\trule {\n\t\t\taction = \"replace\"\n\t\t\tsource_labels = [\"__meta_kubernetes_pod_name\"]\n\t\t\ttarget_label = \"pod\"\n\t\t}\n\n\t\trule @@ -571,7 +915,7 @@ data: {\n\t\t\t\tenabled = true\n\t\t\t}\n\t\t}\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 namespace: monitoring-system --- apiVersion: v1 @@ -581,40 +925,58 @@ data: Transformation\ndeclare \"process_and_transform\" {\n\n\t/*****************************************************************\n\t* ARGUMENTS\n\t*****************************************************************/\n\targument \"traces_forward_to\" {\n\t\tcomment = \"Must be a list(TracesReceiver) where - collected traces should be forwarded to\"\n\t}\n\n\targument \"logs_forward_to\" - {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected logs should be - forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment = \"Must - be a list(MetricsReceiver) where collected metrics should be forwarded to\"\n\t}\n\n\targument - \"cluster\" {\n\t\toptional = true\n\t\tdefault = \"k3d-k3s-codelab\"\n\t}\n\n\targument - \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4318\"\n\t}\n\n\targument - \"otlp_grpc_endpoint\" {\n\t\toptional = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* + collected traces should be forwarded to\"\n\t}\n\n\targument \"cluster\" { }\n\n\targument + \"logs_forward_to\" {\n\t\tcomment = \"Must be a list(LogsReceiver) where collected + logs should be forwarded to\"\n\t}\n\n\targument \"metrics_forward_to\" {\n\t\tcomment + = \"Must be a list(MetricsReceiver) where collected metrics should be forwarded + to\"\n\t}\n\n\targument \"otlp_http_endpoint\" {\n\t\toptional = true\n\t\tdefault + \ = \"0.0.0.0:4318\"\n\t}\n\n\targument \"otlp_grpc_endpoint\" {\n\t\toptional + = true\n\t\tdefault = \"0.0.0.0:4317\"\n\t}\n\n\t/*****************************************************************\n\t* Jaeger for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.jaeger \"default\" {\n\t\tprotocols {\n\t\t\tgrpc {\n\t\t\t\tendpoint = \"0.0.0.0:14250\"\n\t\t\t}\n\n\t\t\tthrift_http {\n\t\t\t\tendpoint = \"0.0.0.0:14268\"\n\t\t\t}\n\n\t\t\tthrift_binary {\n\t\t\t\tendpoint = \"0.0.0.0:6832\"\n\t\t\t}\n\n\t\t\tthrift_compact {\n\t\t\t\tendpoint = \"0.0.0.0:6831\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [otelcol.processor.resourcedetection.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* + {\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\t/*****************************************************************\n\t* Otelcol for Metrics Logs Traces\n\t*****************************************************************/\n\totelcol.receiver.otlp - \"default\" {\n\t\tgrpc {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp - {\n\t\t\tendpoint = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics - = [otelcol.processor.batch.default.input]\n\t\t\tlogs = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces - \ = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection - \"default\" {\n\t\tdetectors = [\"env\"]\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces - = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes + \"default\" {\n\t\tdebug_metrics {\n\t\t\tdisable_high_cardinality_metrics = true\n\t\t}\n\n\t\tgrpc + {\n\t\t\tendpoint = argument.otlp_grpc_endpoint.value\n\t\t}\n\n\t\thttp {\n\t\t\tendpoint + = argument.otlp_http_endpoint.value\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.resourcedetection.default.input]\n\t\t\tlogs + \ = [otelcol.processor.resourcedetection.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.resourcedetection.default.input,\n\t\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.processor.resourcedetection + \"default\" {\n\t\tdetectors = [\"env\", \"system\"]\n\n\t\tsystem {\n\t\t\thostname_sources + = [\"os\"]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input]\n\n\t\t\tlogs + \ = [otelcol.processor.k8sattributes.default.input]\n\t\t\ttraces = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform + \"add_metric_datapoint_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements + {\n\t\t\tcontext = \"datapoint\"\n\t\t\tstatements = [\n\t\t\t\t\"set(attributes[\\\"deployment.environment\\\"], + resource.attributes[\\\"deployment.environment\\\"])\",\n\t\t\t\t\"set(attributes[\\\"service.version\\\"], + resource.attributes[\\\"service.version\\\"])\",\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.k8sattributes \"default\" {\n\t\textract {\n\t\t\tmetadata = [\n\t\t\t\t\"k8s.namespace.name\",\n\t\t\t\t\"k8s.pod.name\",\n\t\t\t\t\"k8s.deployment.name\",\n\t\t\t\t\"k8s.statefulset.name\",\n\t\t\t\t\"k8s.daemonset.name\",\n\t\t\t\t\"k8s.cronjob.name\",\n\t\t\t\t\"k8s.job.name\",\n\t\t\t\t\"k8s.node.name\",\n\t\t\t\t\"k8s.pod.uid\",\n\t\t\t\t\"k8s.pod.start_time\",\n\t\t\t]\n\t\t}\n\n\t\tpod_association {\n\t\t\tsource {\n\t\t\t\tfrom = \"connection\"\n\t\t\t}\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t\ttraces - = [otelcol.processor.transform.add_resource_attributes.input]\n\t\t}\n\t}\n\n\totelcol.processor.transform - \"add_resource_attributes\" {\n\t\terror_mode = \"ignore\"\n\n\t\tlog_statements + {\n\t\t\tmetrics = [otelcol.processor.transform.default.input]\n\t\t\tlogs = + [otelcol.processor.transform.default.input]\n\t\t\ttraces = [\n\t\t\t\totelcol.processor.transform.default.input,\n\t\t\t\totelcol.connector.host_info.default.input,\n\t\t\t]\n\t\t}\n\t}\n\n\totelcol.connector.host_info + \"default\" {\n\t\thost_identifiers = [\"k8s.node.name\"]\n\n\t\toutput {\n\t\t\tmetrics + = [otelcol.processor.batch.host_info_batch.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + \"host_info_batch\" {\n\t\toutput {\n\t\t\tmetrics = [otelcol.exporter.prometheus.host_info_metrics.input]\n\t\t}\n\t}\n\n\totelcol.exporter.prometheus + \"host_info_metrics\" {\n\t\tadd_metric_suffixes = false\n\t\tforward_to = + argument.metrics_forward_to.value\n\t}\n\n\totelcol.processor.transform \"default\" + {\n\t\terror_mode = \"ignore\"\n\n\t\tmetric_statements {\n\t\t\tcontext = + \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\tlog_statements {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])`,\n\t\t\t\t`set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])`,\n\t\t\t\t`set(attributes[\"loki.resource.labels\"], \"pod, namespace, cluster, job\")`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements - {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], - \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\toutput - {\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter - \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\toutput {\n\t\t\tlogs = [otelcol.processor.batch.default.input]\n\t\t\ttraces - = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch + {\n\t\t\tcontext = \"resource\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t\t`set(attributes[\"k8s.cluster.name\"], + \"k3d-k3s-codelab\") where attributes[\"k8s.cluster.name\"] == nil`,\n\t\t\t]\n\t\t}\n\n\t\ttrace_statements + {\n\t\t\tcontext = \"span\"\n\t\t\tstatements = [\n\t\t\t\t`limit(attributes, + 100, [])`,\n\t\t\t\t`truncate_all(attributes, 4096)`,\n\t\t\t]\n\t\t}\n\n\t\toutput + {\n\t\t\tmetrics = [otelcol.processor.filter.default.input]\n\t\t\tlogs = [otelcol.processor.filter.default.input]\n\t\t\ttraces + \ = [otelcol.processor.filter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.filter + \"default\" {\n\t\terror_mode = \"ignore\"\n\n\t\ttraces {\n\t\t\tspan = [\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/live\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] == \\\"/healthy\\\"\",\n\t\t\t\t\"attributes[\\\"http.route\\\"] + == \\\"/ready\\\"\",\n\t\t\t]\n\t\t}\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.batch.default.input]\n\t\t\tlogs + \ = [otelcol.processor.batch.default.input]\n\t\t\ttraces = [otelcol.processor.batch.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.batch \"default\" {\n\t\tsend_batch_size = 16384\n\t\tsend_batch_max_size = 0\n\t\ttimeout \ = \"5s\"\n\n\t\toutput {\n\t\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\t\tlogs \ = [otelcol.processor.memory_limiter.default.input]\n\t\t\ttraces = [otelcol.processor.memory_limiter.default.input]\n\t\t}\n\t}\n\n\totelcol.processor.memory_limiter @@ -650,7 +1012,7 @@ data: \"alloy_traces_input\" {\n\t\tvalue = otelcol.processor.batch.default.input\n\t}\n}\n" kind: ConfigMap metadata: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 namespace: monitoring-system --- apiVersion: v1 @@ -1312,19 +1674,19 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-55dd47md5f + name: alloy-config-dkdkkgdd64 name: config - configMap: - name: alloy-modules-kubernetes-metrics-44gc54b647 + name: alloy-modules-kubernetes-metrics-db96446m8f name: modules-kubernetes-metrics - configMap: - name: alloy-modules-kubernetes-logs-gd277tmt9h + name: alloy-modules-kubernetes-logs-kd42m6dc8k name: modules-kubernetes-logs - configMap: - name: alloy-modules-kubernetes-traces-8mgm8th9m5 + name: alloy-modules-kubernetes-traces-4h9946thb4 name: modules-kubernetes-traces - configMap: - name: alloy-modules-kubernetes-profiles-66c27bc84g + name: alloy-modules-kubernetes-profiles-8hhkt7m7k9 name: modules-kubernetes-profiles --- apiVersion: monitoring.coreos.com/v1 diff --git a/kubernetes/read-write-mode/metrics/metrics.alloy b/kubernetes/read-write-mode/metrics/metrics.alloy index 606728c9..b37ad416 100644 --- a/kubernetes/read-write-mode/metrics/metrics.alloy +++ b/kubernetes/read-write-mode/metrics/metrics.alloy @@ -24,16 +24,13 @@ import.file "metrics" { filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/metrics" } -metrics.podmonitors_scrape "kubernetes" { +metrics.annotations_scrape "kubernetes" { forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] + + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + scrape_interval = "15s" } metrics.servicemonitors_scrape "kubernetes" { forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] } - -// metrics.annotations_scrape "kubernetes" { -// label_prefix = "metrics.grafana.com" -// forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] -// scrape_interval = "15s" -// }