diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/README.md b/kubernetes/common/grafana-agent/configs/modules/kubernetes/README.md new file mode 100644 index 00000000..9afd8f64 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/README.md @@ -0,0 +1,73 @@ +# Kubernetes Modules + +## Logs + +The following pod annotations are supported: + +| Annotation | Description | +| :--------------- | :-----------| +| `logs.agent.grafana.com/scrape` | Allow a pod to declare it's logs should be dropped. | +| `logs.agent.grafana.com/tenant` | Allow a pod to override the tenant for its logs. | +| `logs.agent.grafana.com/log-format` | If specified additional processing is performed to extract details based on the specified format. This value can be a comma-delimited list, in the instances a pod may have multiple containers. The following formats are currently supported: | +| `logs.agent.grafana.com/scrub-level` | Boolean whether or not the level should be dropped from the log message (as it is a label). | +| `logs.agent.grafana.com/scrub-timestamp` | Boolean whether or not the timestamp should be dropped from the log message (as it is metadata). | +| `logs.agent.grafana.com/scrub-nulls` | Boolean whether or not keys with null values should be dropped from json, reducing the size of the log message. | +| `logs.agent.grafana.com/scrub-empties` | Boolean whether or not keys with empty values (`"", [], {}`) should be dropped from json, reducing the size of the log message. | +| `logs.agent.grafana.com/embed-pod` | Boolean whether or not to inject the name of the pod to the end of the log message i.e. `__pod=agent-logs-grafana-agent-jrqms`. | +| `logs.agent.grafana.com/drop-info` | Boolean whether or not info messages should be dropped (default is `false`), but a pod can override this temporarily or permanently. | +| `logs.agent.grafana.com/drop-debug` | Boolean whether or not debug messages should be dropped (default is `true`), but a pod can override this temporarily or permanently. | +| `logs.agent.grafana.com/drop-trace` | Boolean whether or not trace messages should be dropped (default is `true`), but a pod can override this temporarily or permanently. | +| `logs.agent.grafana.com/mask-ssn` | Boolean whether or not to mask SSNs in the log line, if true the data will be masked as `*SSN*salt*` | +| `logs.agent.grafana.com/mask-credit-card` | Boolean whether or not to mask credit cards in the log line, if true the data will be masked as `*credit-card*salt*` | +| `logs.agent.grafana.com/mask-email` | Boolean whether or not to mask emails in the log line, if true the data will be masked as`*email*salt*` | +| `logs.agent.grafana.com/mask-ipv4` | Boolean whether or not to mask IPv4 addresses in the log line, if true the data will be masked as`*ipv4*salt*` | +| `logs.agent.grafana.com/mask-ipv6` | Boolean whether or not to mask IPv6 addresses in the log line, if true the data will be masked as `*ipv6*salt*` | +| `logs.agent.grafana.com/mask-phone` | Boolean whether or not to mask phone numbers in the log line, if true the data will be masked as `*phone*salt*` | + +--- +## Metrics + +The following pod annotations are supported are supported for gathering of metrics for pods and endpoints: + +| Annotation | Description | +| :--------------- | :-----------| +| `metrics.agent.grafana.com/scrape`
`prometheus.io/scrape` | Boolean whether or not to scrape the endpoint / pod for metrics. *Note*: If a pod exposes multiple ports, all ports would be scraped for metrics. To limit this behavior specify the port annotation to limit the scrape to a single port. If the label `prometheus.io/service-monitor` or `metrics.agent.grafana.com/service-monitor` is set to `"false"` that is interpreted as a `scrape: "false"` | +| `metrics.agent.grafana.com/scheme`
`prometheus.io/scheme` | The default scraping scheme is `http`, this can be specified as a single value which would override, the schema being used for all ports attached to the endpoint / pod. | +| `metrics.agent.grafana.com/path`
`prometheus.io/path` | The default path to scrape is `/metrics`, this can be specified as a single value which would override, the scrape path being used for all ports attached to the endpoint / pod. | +| `metrics.agent.grafana.com/port`
`prometheus.io/port` | The default port to scrape is the endpoint port, this can be specified as a single value which would override the scrape port being used for all ports attached to the endpoint, note that even if aan endpoint had multiple targets, the relabel_config targets are deduped before scraping | +| `metrics.agent.grafana.com/tenant` | The tenant their metrics should be sent to, this does not necessarily have to be the actual tenantId, it can be a friendly name as well that is simply used to determine if the metrics should be gathered for the current tenant | +| `metrics.agent.grafana.com/job` | The job label value to use when collecting their metrics, this can be useful as endpoints/pods will be automatically scraped for metrics, separate jobs do not have to be defined. However, it is common to use an integration or community project where rules / dashboards are provided for you. Oftentimes, this provided assets use hard-coded values for a job label i.e. `...{job="integrations/kubernetes/cadvisor"...}` or `...{job="kube-state-metrics"...}` setting this annotation to that value will allow the provided asset to work out of the box. | +| `metrics.agent.grafana.com/interval`
`prometheus.io/interval` | The default interval to scrape is `1m`, this can be specified as a single value which would override, the scrape interval being used for all ports attached to the endpoint / pod. | +| `metrics.agent.grafana.com/timeout`
`prometheus.io/timeout` | The default timeout for scraping is `10s`, this can be specified as a single value which would override, the scrape interval being used for all ports attached to the endpoint / pod. | + +### Probes (Blackbox) + +The following service / ingress annotations are supported are supported for probes and the gathering of metrics from blackbox exporter: + +| Annotation | Description | +| :--------------- | :-----------| +| `probes.agent.grafana.com/probe`
`prometheus.io/probe` | Boolean whether or not to probe the service / ingress for metrics. *Note*: If a pod exposes multiple ports, all ports would be probed. To limit this behavior specify the port annotation to limit the probe to a single port. | +| `probes.agent.grafana.com/port`
`prometheus.io/port` | The default port to probe is the service / ingress port, this can be specified as a single value which would override the probe port being used for all ports attached to the service / ingress, note that even if aan service / ingress had multiple targets, the relabel_config targets are deduped before scraping | +| `probes.agent.grafana.com/path`
`prometheus.io/path` | The default path to probe is `/metrics`, this can be specified as a single value which would override, the probe path being used for all ports attached to the service / ingress. | +| `probes.agent.grafana.com/module`
`prometheus.io/module` | The name of the blackbox module to use for probing the resource, the default value is "unknown" as these values should be determined from your blackbox-exporter configuration file. | +| `probes.agent.grafana.com/tenant` | The tenant their metrics should be sent to, this does not necessarily have to be the actual tenantId, it can be a friendly name as well that is simply used to determine if the metrics should be gathered for the current tenant | +| `probes.agent.grafana.com/job` | The job label value to use when collecting their metrics, this can be useful as service / ingress will be automatically probed for metrics, separate jobs do not have to be defined. However, it is common to use an integration or community project where rules / dashboards are provided for you. Oftentimes, this provided assets use hard-coded values for a job label i.e. `...{job="blackbox-exporter"...}` setting this annotation to that value will allow the provided asset to work out of the box. | +| `probes.agent.grafana.com/interval` | The default interval to probe is `1m`, this can be specified as a single value which would override, the probe interval being used for all ports attached to the service / ingress. | +| `probes.agent.grafana.com/timeout` | The default timeout for scraping is `10s`, this can be specified as a single value which would override, the probe interval being used for all ports attached to the service / ingress. | + +### Probes (json-exporter) + +The following service / ingress annotations are supported are supported for probes and the gathering of metrics from json-exporter exporter: + +| Annotation | Description | +| :--------------- | :-----------| +| `json.agent.grafana.com/probe` | Boolean whether or not to probe the service / ingress for metrics. *Note*: If a pod exposes multiple ports, all ports would be probed. To limit this behavior specify the port annotation to limit the probe to a single port. | +| `json.agent.grafana.com/port`
`prometheus.io/port` | The default port to probe is the service / ingress port, this can be specified as a single value which would override the probe port being used for all ports attached to the service / ingress, note that even if aan service / ingress had multiple targets, the relabel_config targets are deduped before scraping | +| `json.agent.grafana.com/path` | The default path to probe is `/metrics`, this can be specified as a single value which would override, the probe path being used for all ports attached to the service / ingress. | +| `json.agent.grafana.com/module` | The name of the json-exporter module to use for probing the resource, the default value is "unknown" as these values should be determined from your json-exporter-exporter configuration file. | +| `json.agent.grafana.com/tenant` | The tenant their metrics should be sent to, this does not necessarily have to be the actual tenantId, it can be a friendly name as well that is simply used to determine if the metrics should be gathered for the current tenant | +| `json.agent.grafana.com/job` | The job label value to use when collecting their metrics, this can be useful as service / ingress will be automatically probed for metrics, separate jobs do not have to be defined. However, it is common to use an integration or community project where rules / dashboards are provided for you. Oftentimes, this provided assets use hard-coded values for a job label i.e. `...{job="json-exporter"...}` setting this annotation to that value will allow the provided asset to work out of the box. | +| `json.agent.grafana.com/interval` | The default interval to probe is `1m`, this can be specified as a single value which would override, the probe interval being used for all ports attached to the service / ingress. | +| `json.agent.grafana.com/timeout` | The default timeout for scraping is `10s`, this can be specified as a single value which would override, the probe interval being used for all ports attached to the service / ingress. | + +See [/example/kubernetes/metrics](../../example/kubernetes/metrics/) for working example configurations. diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/all.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/all.river new file mode 100644 index 00000000..8b5ee25f --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/all.river @@ -0,0 +1,198 @@ +/* +Module: log-all +Description: Wrapper module to include all kubernetes logging modules and use cri parsing +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +argument "tenant" { + // comment = "The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this can be a regex." + optional = true + default = ".*" +} + +argument "keep_labels" { + // comment = "List of labels to keep before the log message is written to Loki" + optional = true + default = [ + "app", + "cluster", + "component", + "container", + "deployment", + "env", + "filename", + "instance", + "job", + "level", + "log_type", + "namespace", + "region", + "service", + "squad", + "team", + ] +} + +argument "git_repo" { + optional = true + default = coalesce(env("GIT_REPO"), "https://github.com/grafana/agent-modules.git") +} + +argument "git_rev" { + optional = true + default = coalesce(env("GIT_REV"), env("GIT_REVISION"), env("GIT_BRANCH"), "main") +} + +argument "git_pull_freq" { + // comment = "How often to pull the git repo, the default is 0s which means never pull" + optional = true + default = "0s" +} + +module.git "log_targets" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/targets/logs-from-worker.river" + + arguments { + forward_to = [module.git.log_formats_all.exports.process.receiver] + tenant = argument.tenant.value + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +module.git "log_formats_all" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/all.river" + + arguments { + forward_to = [module.git.log_level_default.exports.process.receiver] + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +module.git "log_level_default" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/labels/log-level.river" + + arguments { + forward_to = [module.git.label_normalize_filename.exports.process.receiver] + } +} + +module.git "label_normalize_filename" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/labels/normalize-filename.river" + + arguments { + // here we fork, one branch goes to the log level module, the other goes to the metrics module + // this is because we need to reduce the labels on the pre-metrics but they are still necessary in + // downstream modules + forward_to = [ + module.git.pre_process_metrics.exports.process.receiver, + module.git.drop_levels.exports.process.receiver, + ] + } +} + +module.git "pre_process_metrics" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/metrics/pre-process-bytes-lines.river" + + arguments { + forward_to = [module.git.drop_levels.exports.process.receiver] + keep_labels = argument.keep_labels.value + } +} + +module.git "drop_levels" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/drops/levels.river" + + arguments { + forward_to = [module.git.scrub_all.exports.process.receiver] + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +module.git "scrub_all" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/scrubs/all.river" + + arguments { + forward_to = [module.git.embed_pod.exports.process.receiver] + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +module.git "embed_pod" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/embed/pod.river" + + arguments { + forward_to = [module.git.mask_all.exports.process.receiver] + } +} + +module.git "mask_all" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/masks/all.river" + + arguments { + forward_to = [module.git.label_keep.exports.process.receiver] + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +module.git "label_keep" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/labels/keep-labels.river" + + arguments { + forward_to = [module.git.post_process_metrics.exports.process.receiver] + keep_labels = argument.keep_labels.value + } +} + +module.git "post_process_metrics" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/metrics/post-process-bytes-lines.river" + + arguments { + forward_to = argument.forward_to.value + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/drops/level-debug.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/drops/level-debug.river new file mode 100644 index 00000000..0d3b63a3 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/drops/level-debug.river @@ -0,0 +1,28 @@ +/* +Module: drop-debug +Description: The default behavior is to drop debug level messaging automatically, however, debug level + messages can still be logged by adding the annotation: + + logs.agent.grafana.com/drop-debug: false +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.drop_debug +} + +loki.process "drop_debug" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/drop-debug annotation, if not set or set to true then drop + // any log message with level=debug + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/drop-debug: true" + selector = "{level=~\"(?i)debug?\",logs_agent_grafana_com_drop_debug!=\"false\"}" + action = "drop" + drop_counter_reason = "debug" + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/drops/level-info.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/drops/level-info.river new file mode 100644 index 00000000..4173934e --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/drops/level-info.river @@ -0,0 +1,28 @@ +/* +Module: drop-info +Description: The default behavior is to keep info level messaging automatically, however, info level + messages can dropped by adding the annotation: + + logs.agent.grafana.com/drop-info: true +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.drop_info +} + +loki.process "drop_info" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/drop-info annotation, if not set or set to true then drop + // any log message with level=info + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/drop-info: true" + selector = "{level=~\"(?i)info?\",logs_agent_grafana_com_drop_info=\"true\"}" + action = "drop" + drop_counter_reason = "info" + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/drops/level-trace.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/drops/level-trace.river new file mode 100644 index 00000000..6d62e77d --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/drops/level-trace.river @@ -0,0 +1,28 @@ +/* +Module: drop-trace +Description: The default behavior is to drop trace level messaging automatically, however, trace level + messages can still be logged by adding the annotation: + + logs.agent.grafana.com/drop-trace: false +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.drop_trace +} + +loki.process "drop_trace" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/drop-trace annotation, if not set or set to true then drop + // any log message with level=trace + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/drop-trace: true" + selector = "{level=~\"(?i)trace?\",logs_agent_grafana_com_drop_trace!=\"false\"}" + action = "drop" + drop_counter_reason = "trace" + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/drops/levels.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/drops/levels.river new file mode 100644 index 00000000..55d71101 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/drops/levels.river @@ -0,0 +1,60 @@ +/* +Module: drop-levels +Description: Wrapper module to include all drop level modules +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +argument "git_repo" { + optional = true + default = coalesce(env("GIT_REPO"), "https://github.com/grafana/agent-modules.git") +} + +argument "git_rev" { + optional = true + default = coalesce(env("GIT_REV"), env("GIT_REVISION"), env("GIT_BRANCH"), "main") +} + +argument "git_pull_freq" { + optional = true + default = "5m" +} + +export "process" { + value = module.git.drop_trace.exports.process +} + +module.git "drop_trace" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/drops/level-trace.river" + + arguments { + forward_to = [module.git.drop_debug.exports.process.receiver] + } +} + +module.git "drop_debug" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/drops/level-debug.river" + + arguments { + forward_to = [module.git.drop_info.exports.process.receiver] + } +} + +module.git "drop_info" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/drops/level-info.river" + + arguments { + forward_to = argument.forward_to.value + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/embed/pod.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/embed/pod.river new file mode 100644 index 00000000..939b67ae --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/embed/pod.river @@ -0,0 +1,60 @@ +/* +Module: embed-pod +Description: Embeds the name of the pod to the json or text log line +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.embed_pod +} + +loki.process "embed_pod" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/embed-pod annotation, if true embed the name of the pod to the end of the log line + // this can reduce the overall cardinality, by not using a label of "pod", individual pods can still be searched + // using a line selector i.e. __pod=your-pod-name + stage.match { + selector = "{logs_agent_grafana_com_embed_pod=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/embed-pod: true" + + // embed as json property + stage.match { + selector = "{logs_agent_grafana_com_log_format=~\"(?i)(.*json|istio|otel|open-?telemetry)\"}" + // render a new label called log_line, and add the name of the pod to the end of the log message + // knowing the pod name can be valuable for debugging, but it should not be a label in Loki due + // to the high cardinality it would create. + // note: .Entry is a special key that is used to reference the current line + stage.replace { + expression = "\\}$" + replace = "" + } + + stage.template { + source = "log_line" + template = "{{ .Entry }},\"__pod\":\"{{ .pod }}\"}" + } + } + + // embed as text property + stage.match { + selector = "{logs_agent_grafana_com_log_format!~\"(?i)(.*json|istio|otel|open-?telemetry)\"}" + // render a new label called log_line, and add the name of the pod to the end of the log message + // knowing the pod name can be valuable for debugging, but it should not be a label in Loki due + // to the high cardinality it would create. + // note: .Entry is a special key that is used to reference the current line + stage.template { + source = "log_line" + template = "{{ .Entry }} __pod={{ .pod }}" + } + } + + // reset the output to the log_line + stage.output { + source = "log_line" + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/events.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/events.river new file mode 100644 index 00000000..085c77b8 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/events.river @@ -0,0 +1,209 @@ +/* +Module: logs-kubernetes-events +Description: Retrieves and processes kubernetes events. This module should only be deployed as part of a StatefulSet with + a single replica. As events are captured from API calls, any more than a single pod would result in duplicate + events being sent to Loki. +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +argument "namespaces" { + // comment = "List of namespaces to collect events from" + optional = true + default = [] +} + +argument "label_job" { + // comment = "The job label to set for all collected logs" + optional = true + default = "integrations/kubernetes/eventhandler" +} + +argument "keep_labels" { + // comment = "List of labels to keep before the log message is written to Loki" + optional = true + default = [ + "app", + "cluster", + "component", + "env", + "instance", + "job", + "level", + "log_type", + "region", + "squad", + "team", + ] +} + +argument "drop_debug" { + // comment = "Whether or not to drop debug messages" + optional = true + default = "true" +} + +argument "drop_info" { + // comment = "Whether or not to drop info messages" + optional = true + default = "false" +} + +argument "git_repo" { + optional = true + default = coalesce(env("GIT_REPO"), "https://github.com/grafana/agent-modules.git") +} + +argument "git_rev" { + optional = true + default = coalesce(env("GIT_REV"), env("GIT_REVISION"), env("GIT_BRANCH"), "main") +} + +argument "git_pull_freq" { + optional = true + default = "5m" +} + +loki.source.kubernetes_events "events" { + job_name = argument.label_job.value + namespaces = argument.namespaces.value + forward_to = [loki.relabel.events.receiver] +} + +loki.relabel "events" { + forward_to = [module.git.log_format_logfmt.exports.process.receiver] + + // events are in logfmt format + rule { + action = "replace" + replacement = "logfmt" + target_label = "logs_agent_grafana_com_log_format" + } + + // default to log level info as events don't always have a level + rule { + action = "replace" + replacement = "Info" + target_label = "level" + } + + // set the annotations for drop_debug and drop_info + rule { + action = "replace" + replacement = argument.drop_debug.value + target_label = "logs_agent_grafana_com_drop_debug" + } + + rule { + action = "replace" + replacement = argument.drop_info.value + target_label = "logs_agent_grafana_com_drop_info" + } +} + +module.git "log_format_logfmt" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/logfmt.river" + + arguments { + forward_to = [loki.process.events.receiver] + } +} + +// set the instance label to the source host +loki.process "events" { + forward_to = [module.git.drop_levels.exports.process.receiver] + + stage.logfmt { + mapping = { + "component" = "sourcecomponent", + "instance" = "sourcehost", + // most events don't have a level but they do have a "type" i.e. Normal, Warning, Error, etc. + "level" = "type", + } + } + + // set the instance extracted value as a label + stage.labels { + values = { + component = "", + instance = "", + level = "", + } + } + + // since we're using the level label derived from the "type" property, one type is "Normal" which is not a log level + // and will not properly color to an info level message, set the level to info. This is also relavent for if dropping + // info level messages is desired. As Normal will contain messages about pods starting, stopping, etc. which may be + // something undesirable to keep. + stage.match { + selector = "{level=\"Normal\"}" + + stage.static_labels { + values = { + level = "Info", + } + } + } + + // if instance is not set then look for the assigned keyword as it may be part of the msg string + // look for certain keywords and set a log level based on that + // i.e. msg="Successfully assigned agents/blackbox-prometheus-blackbox-exporter-696d8fcd54-8qgz5 to gke-dev-default-pool-0a5344ff-0eea" + stage.match { + selector = "{instance=~\".*module.git.*|\"}" + + // while the level could be extracted as logfmt, this allows for multiple possible log levels formats + // i.e. loglevel=info, level=info, lvl=info, loglvl=info + stage.regex { + // unescaped regex: (?:".+ assigned \w+\/(\w+|-)+ to )(?P[^ "]+)" + expression = "(?:\".+ assigned \\w+\\/(\\w+|-)+ to )(?P[^ \"]+)\"" + } + + // set the extracted level value as a label + stage.labels { + values = { + instance = "", + } + } + } + + // if instance is still not set, drop the label because its value is something similar to "module.git.event_logs/loki.source.kubernetes_events.events" + // which is not useful + stage.match { + selector = "{instance=~\".*module.git.*|\"}" + + stage.label_drop { + values = ["instance"] + } + } +} + +module.git "drop_levels" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/drops/levels.river" + + arguments { + forward_to = [module.git.label_keep.exports.process.receiver] + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +module.git "label_keep" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/labels/keep-labels.river" + + arguments { + forward_to = argument.forward_to.value + keep_labels = argument.keep_labels.value + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/kubelet.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/kubelet.river new file mode 100644 index 00000000..56770a15 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/kubelet.river @@ -0,0 +1,234 @@ +/* +Module: log-kubelet +Description: Retrieves and processes the systemd journal logs for the kubelet +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +argument "journal_max_age" { + // comment = "The oldest relative time from process start that will be read." + optional = true + default = "12h" +} + +argument "journal_path" { + // comment = "The path to the journal files" + optional = true + default = "/var/log/journal" +} + +argument "journal_filter" { + // comment = "Filter the system unit files to read from the journal" + optional = true + default = ".+" +} + +argument "scrub_level" { + // comment = "Whether or not to scrub the log level from the log line" + optional = true + default = "false" +} + +argument "scrub_timestamp" { + // comment = "Whether or not to scrub the timestamp from the log line" + optional = true + default = "false" +} + +argument "drop_debug" { + // comment = "Whether or not to drop debug messages" + optional = true + default = "true" +} + +argument "label_job" { + // comment = "The job label to set for all collected logs" + optional = true + default = "loki.source.journal.kubelet" +} + +argument "keep_labels" { + // comment = "List of labels to keep before the log message is written to Loki" + optional = true + default = [ + "app", + "cluster", + "env", + "instance", + "job", + "level", + "log_type", + "region", + "squad", + "unit", + "team", + ] +} + +argument "git_repo" { + optional = true + default = coalesce(env("GIT_REPO"), "https://github.com/grafana/agent-modules.git") +} + +argument "git_rev" { + optional = true + default = coalesce(env("GIT_REV"), env("GIT_REVISION"), env("GIT_BRANCH"), "main") +} + +argument "git_pull_freq" { + optional = true + default = "5m" +} + +loki.relabel "journal" { + forward_to = [] + + // filter unit files + rule { + action = "keep" + source_labels = ["__journal__systemd_unit"] + regex = argument.journal_filter.value + } + + // copy all journal labels and make the available to the pipeline stages as labels, there is a label keep defined to filter + // out unwanted labels, the following labels are available: + // - boot_id + // - cap_effective + // - cmdline + // - comm + // - exe + // - gid + // - hostname + // - machine_id + // - pid + // - stream_id + // - systemd_cgroup + // - systemd_invocation_id + // - systemd_slice + // - systemd_unit + // - transport + // - uid + // + // More Info: https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html + rule { + action = "labelmap" + regex = "__journal__(.+)" + } + + rule { + action = "replace" + source_labels = ["__journal__systemd_unit"] + replacement = "$1" + target_label = "unit" + } + + // kubelet logs are in klog format + rule { + action = "replace" + source_labels = ["__journal__systemd_unit"] + regex = "(kubelet|node-problem-detector)\\.service" + replacement = "klog" + target_label = "logs_agent_grafana_com_log_format" + } + + // containerd logs are in logfmt format + rule { + action = "replace" + source_labels = ["__journal__systemd_unit"] + regex = "(containerd).service" + replacement = "logfmt" + target_label = "logs_agent_grafana_com_log_format" + } +} + +loki.source.journal "kubelet" { + max_age = argument.journal_max_age.value + path = argument.journal_path.value + forward_to = [module.git.log_format_klog.exports.process.receiver] + relabel_rules = loki.relabel.journal.rules + labels = { + job = argument.label_job.value, + // set the log format + logs_agent_grafana_com_log_format = "syslog", + // set whether or not to scrub the timestamp + logs_agent_grafana_com_scrub_timestamp = argument.scrub_timestamp.value, + // set whether or not to scrub the log level + logs_agent_grafana_com_scrub_level = argument.scrub_level.value, + // set whether or not to drop debug level messages + logs_agent_grafana_com_log_format = argument.drop_debug.value, + // set an instance label to be the hostname + instance = env("HOSTNAME"), + } +} + +module.git "log_format_klog" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/klog.river" + + arguments { + forward_to = [module.git.log_format_logfmt.exports.process.receiver] + } +} + +module.git "log_format_logfmt" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/logfmt.river" + + arguments { + forward_to = [module.git.log_format_syslog.exports.process.receiver] + } +} + +module.git "log_format_syslog" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/syslog.river" + + arguments { + forward_to = [module.git.log_level_default.exports.process.receiver] + } +} + +module.git "log_level_default" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/labels/log-level.river" + + arguments { + forward_to = [module.git.drop_levels.exports.process.receiver] + } +} + +module.git "drop_levels" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/drops/levels.river" + + arguments { + forward_to = [module.git.label_keep.exports.process.receiver] + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +module.git "label_keep" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/labels/keep-labels.river" + + arguments { + forward_to = argument.forward_to.value + keep_labels = argument.keep_labels.value + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/labels/keep-labels.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/labels/keep-labels.river new file mode 100644 index 00000000..e39bd395 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/labels/keep-labels.river @@ -0,0 +1,52 @@ +/* +Module: keep-labels +Description: Pre-defined set of labels to keep, this stage should always be in-place as the previous relabeing + stages make every pod label and annotation a label in the pipeline, which we do not want created + in Loki as that would have extremely high-cardinality. +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +argument "keep_labels" { + optional = true + // comment = "List of labels to keep before the log message is written to Loki" + default = [ + "app", + "cluster", + "component", + "container", + "deployment", + "env", + "filename", + "instance", + "job", + "level", + "log_type", + "namespace", + "region", + "service", + "squad", + "team", + ] +} + +export "process" { + value = loki.process.keep_labels +} + +/* +As all of the pod labels and annotations we transformed into labels in the previous relabelings to make +them available to the pipeline processing we need to ensure they are not automatically created in Loki. +This would result in an extremely high number of labels and values severely impacting query performance. +Not every log has to contain these labels, but this list should reflect the set of labels that you want +to explicitly allow. +*/ +loki.process "keep_labels" { + forward_to = argument.forward_to.value + + stage.label_keep { + values = argument.keep_labels.value + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/labels/log-level.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/labels/log-level.river new file mode 100644 index 00000000..f4af22fe --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/labels/log-level.river @@ -0,0 +1,105 @@ +/* +Module: log-level +Description: Sets a default log level of "unknown", then based on known patterns attempts to assign an appropriate log + log level based on the contents of the log line. This should be considered as default/initial processing + as there are modules for parsing specific log patterns. +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_level +} + +loki.process "log_level" { + forward_to = argument.forward_to.value + + // if a log level is not set, default it to unknown + stage.match { + selector = "{level=\"\"}" + + // default level to unknown + stage.static_labels { + values = { + level = "unknown", + } + } + } + + // if a log_type is not set, default it to unknown + stage.match { + selector = "{log_type=\"\"}" + + // default level to unknown + stage.static_labels { + values = { + log_type = "unknown", + } + } + } + + // check to see if the log line matches the klog format (https://github.com/kubernetes/klog) + stage.match { + // unescaped regex: ([IWED][0-9]{4}\s+[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+) + selector = "{level=\"unknown\"} |~ \"([IWED][0-9]{4}\\\\s+[0-9]{2}:[0-9]{2}:[0-9]{2}\\\\.[0-9]+)\"" + + // extract log level, klog uses a single letter code for the level followed by the month and day i.e. I0119 + stage.regex { + expression = "((?P[A-Z])[0-9])" + } + + // if the extracted level is I set INFO + stage.replace { + source = "level" + expression = "(I)" + replace = "INFO" + } + + // if the extracted level is W set WARN + stage.replace { + source = "level" + expression = "(W)" + replace = "WARN" + } + + // if the extracted level is E set ERROR + stage.replace { + source = "level" + expression = "(E)" + replace = "ERROR" + } + + // if the extracted level is I set INFO + stage.replace { + source = "level" + expression = "(D)" + replace = "DEBUG" + } + + // set the extracted level to be a label + stage.labels { + values = { + level = "", + } + } + } + + // if the level is still unknown, do one last attempt at detecting it based on common levels + stage.match { + selector = "{level=\"unknown\"}" + + // unescaped regex: (?i)(?:"(?:level|loglevel|levelname|lvl|SeverityText)":\s*"|\s+(?:level|loglevel|lvl)="?|\s+\[?)(?P(DEBUG?|INFO|WARN(ING)?|ERR(OR)?|CRITICAL|FATAL|NOTICE|TRACE))("|\s+|-|\s*\]) + stage.regex { + expression = "(?i)(?:\"(?:level|loglevel|levelname|lvl|SeverityText)\":\\s*\"|\\s+(?:level|loglevel|lvl)=\"?|\\s+\\[?)(?P(DEBUG?|INFO|WARN(ING)?|ERR(OR)?|CRITICAL|FATAL|NOTICE|TRACE))(\"|\\s+|-|\\s*\\])" + } + + // set the extracted level to be a label + stage.labels { + values = { + level = "", + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/labels/normalize-filename.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/labels/normalize-filename.river new file mode 100644 index 00000000..c2b3ec1b --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/labels/normalize-filename.river @@ -0,0 +1,43 @@ +/* +Module: normalize-filename +Description: Normalizes the kubernetes filename name, and reduces cardinality of the filename +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.normalize_filename +} + +/* +Normalize the filename, the label "filename" is automatically created from +discovered files in the matching path based on the __path__ label from the +relabel_configs. This has extremely high cardinality, it can be useful +for a pod with multiple containers/sidecars to know where the log came from +but we can greatly reduce the cardinality. +Example: + Filename: /var/log/pods/agents_agent-logs-grafana-agent-k8hpm_5cafa323-a7ed-4703-9220-640d3e44a5e3/config-reloader/0.log + Becomes: /var/log/pods/agents/agent-logs-grafana-agent/config-reloader.log +*/ +loki.process "normalize_filename" { + forward_to = argument.forward_to.value + + stage.regex { + // unescaped regex: ^(?P\/([^\/_]+\/)+)[^\/]+\/(?P[^\/]+)\/[0-9]+\.log + expression = "^(?P\\/([^\\/_]+\\/)+)[^\\/]+\\/(?P[^\\/]+)\\/[0-9]+\\.log" + source = "filename" + } + + stage.template { + source = "normalized_filename" + template = "{{ .path }}{{ .job }}/{{ .container_folder }}.log" + } + + stage.labels { + values = { + filename = "normalized_filename", + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/all.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/all.river new file mode 100644 index 00000000..1fb92e7e --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/all.river @@ -0,0 +1,170 @@ +/* +Module: log-format-all +Description: Wrapper module to include all log-format modules +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +argument "git_repo" { + optional = true + default = coalesce(env("GIT_REPO"), "https://github.com/grafana/agent-modules.git") +} + +argument "git_rev" { + optional = true + default = coalesce(env("GIT_REV"), env("GIT_REVISION"), env("GIT_BRANCH"), "main") +} + +argument "git_pull_freq" { + optional = true + default = "5m" +} + +export "process" { + value = module.git.log_format_common_log.exports.process +} + +module.git "log_format_common_log" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/common-log.river" + + arguments { + forward_to = [module.git.log_format_dotnet.exports.process.receiver] + } +} + +module.git "log_format_dotnet" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/dotnet.river" + + arguments { + forward_to = [module.git.log_format_istio.exports.process.receiver] + } +} + +module.git "log_format_istio" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/istio.river" + + arguments { + forward_to = [module.git.log_format_json.exports.process.receiver] + } +} + +module.git "log_format_json" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/json.river" + + arguments { + forward_to = [module.git.log_format_klog.exports.process.receiver] + } +} + +module.git "log_format_klog" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/klog.river" + + arguments { + forward_to = [module.git.log_format_log4j.exports.process.receiver] + } +} + +module.git "log_format_log4j" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/log4j.river" + + arguments { + forward_to = [module.git.log_format_logfmt.exports.process.receiver] + } +} + +module.git "log_format_logfmt" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/logfmt.river" + + arguments { + forward_to = [module.git.log_format_otel.exports.process.receiver] + } +} + +module.git "log_format_otel" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/otel.river" + + arguments { + forward_to = [module.git.log_format_postgres.exports.process.receiver] + } +} + +module.git "log_format_postgres" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/otel.river" + + arguments { + forward_to = [module.git.log_format_python.exports.process.receiver] + } +} + +module.git "log_format_python" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/python.river" + + arguments { + forward_to = [module.git.log_format_spring_boot.exports.process.receiver] + } +} + +module.git "log_format_spring_boot" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/spring-boot.river" + + arguments { + forward_to = [module.git.log_format_syslog.exports.process.receiver] + } +} + +module.git "log_format_syslog" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/syslog.river" + + arguments { + forward_to = [module.git.log_format_zerolog.exports.process.receiver] + } +} + +module.git "log_format_zerolog" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/log-formats/spring-boot.river" + + arguments { + forward_to = argument.forward_to.value + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/common-log.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/common-log.river new file mode 100644 index 00000000..b524fe4a --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/common-log.river @@ -0,0 +1,71 @@ +/* +Module: log-format-clf +Description: Log Processing for common-log (apache/nginx) +Docs: https://www.w3.org/Daemon/User/Config/Logging.html#common-logfile-format +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_format_clf +} + +loki.process "log_format_clf" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains clf and the line matches the format, then process the line as clf + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: clf" + // unescaped regex: \S+\s+\S+\s+\S+\s+\[\S+\s+\S+\]\s+"[^"]+"\s+\d+\s+\d+ + selector = "{log_type=\"\", logs_agent_grafana_com_log_format=~\"(?i).*((apache|nginx|common-?log|clf)).*\"} |~ \"^\\\\S+\\\\s+\\\\S+\\\\s+\\\\S+\\\\s+\\\\[\\\\S+\\\\s+\\\\S+\\\\]\\\\s+\\\"[^\\\"]+\\\"\\\\s+\\\\d+\\\\s+\\\\d+$\"" + + // clf doesn't have a log level, set default to info, set the log_type + stage.static_labels { + values = { + level = "info", + log_type = "clf", + } + } + + // extract the http response code and request method as they might want to be used as labels + stage.regex { + // unescaped regex: (?P\d{3}) "(?P\S+) + expression = "(?P[0-9]{3}) \"(?P\\S+)" + } + + // set the extracted response code and request method as labels + stage.labels { + values = { + response_code = "", + request_method = "", + } + } + + // check to see if the string failed is found in the log line, if so set the level to error + stage.match { + selector = "{logs_agent_grafana_com_log_format=~\"(?i)(apache|nginx|common-log|clf)\"} |~ \" (failed|error) \"" + + stage.static_labels { + values = { + level = "error", + } + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + // unescaped regex: (\[([^\]]+)\]) + stage.replace { + expression = "(\\[([^\\]]+)\\])" + replace = "" + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/dotnet.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/dotnet.river new file mode 100644 index 00000000..7ae1e1b6 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/dotnet.river @@ -0,0 +1,76 @@ +/* +Module: log-format-dotnet +Description: Log Processing for .Net +Docs: https://learn.microsoft.com/en-us/dotnet/core/extensions/console-log-formatter#json +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_format_dotnet +} + +loki.process "log_format_dotnet" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains python-json and the line matches the format, then process the line as python-json + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: dotnet-json" + // unescaped regex: ^\s*\{.+\}\s*$ + selector = "{log_type=\"\", logs_agent_grafana_com_log_format=~\"(?i).*(dotnet-?json).*\"} |~ \"^\\\\s*\\\\{.+\\\\}\\\\s*$\"" + + // set the log_type + stage.static_labels { + values = { + log_type = "dotnet", + } + } + + // extract the level, response_code, method if they exist + stage.json { + expressions = { + level = "LogLevel", + category = "Category", + } + } + + // set the extracted level and category as labels + stage.labels { + values = { + level = "", + category = "", + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + // remove timestamp from the log line, depending on the entry it can be "start_time" or "time" + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + // unescaped regex: (?i)("(Timestamp)"\s*:\s*\[?"[^"]+"\]?,?) + stage.replace { + expression = "(?i)(\"(Timestamp)\"\\s*:\\s*\\[?\"[^\"]+\"\\]?,?)" + replace = "" + } + } + + // check logs.agent.grafana.com/scrub-level annotation, if true remove the level from the log line (it is still a label) + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_level=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-level: true" + + // remove level from the log line + stage.replace { + // unescaped regex: (?i)"LogLevel"\s*:\s*"[^"]+",? + expression = "(?i)(\"LogLevel\"\\s*:\\s*\"[^\"]+\",?)" + replace = "" + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/istio.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/istio.river new file mode 100644 index 00000000..aaa5e99e --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/istio.river @@ -0,0 +1,78 @@ +/* +Module: log-format-istio +Description: Log Processing for istio +Docs: https://istio.io/latest/docs/tasks/observability/logs/access-log/ +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_format_istio +} + +loki.process "log_format_istio" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains istio and the line matches the format, then process the line as json + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: istio" + selector = "{log_type=\"\", logs_agent_grafana_com_log_format=~\"(?i).*(istio-?(json)?).*\"} |~ \"^\\\\s*\\\\{.+\\\\}\\\\s*$\"" + + // not all istio logs contain a level, default to info and set the log_type + stage.static_labels { + values = { + level = "info", + log_type = "istio", + } + } + + // extract the level, response_code, method if they exist + stage.json { + expressions = { + level = "level", + response_code = "response_code", + request_method = "method", + } + } + + // set the extracted level, response code and request method as labels + stage.labels { + values = { + level = "", + response_code = "", + request_method = "", + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + // remove timestamp from the log line, depending on the entry it can be "start_time" or "time" + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + // unescaped regex: ("(start_)?time"\s*:\s*"[^"]+",) + stage.replace { + expression = "(\"(start_)?time\"\\s*:\\s*\"[^\"]+\",)" + replace = "" + } + } + + // check logs.agent.grafana.com/scrub-level annotation, if true remove the level from the log line (it is still a label) + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_level=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-level: true" + + // remove level from the log line + stage.replace { + // unescaped regex: "level"\s*:\s*"[^"]+",? + expression = "(?i)(\"level\"\\s*:\\s*\"[^\"]+\",?)" + replace = "" + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/json.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/json.river new file mode 100644 index 00000000..7777745c --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/json.river @@ -0,0 +1,72 @@ +/* +Module: log-format-json +Description: Log Processing for Generic JSON +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_format_json +} + +loki.process "log_format_json" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains json and the line matches the format, then process the line as json + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: json" + selector = "{log_type=\"\", logs_agent_grafana_com_log_format=~\"(?i).*((generic-?)?json).*\"} |~ \"^\\\\s*\\\\{.+\\\\}\\\\s*$\"" + + // set the log_type + stage.static_labels { + values = { + log_type = "json", + } + } + + // extract the level + stage.json { + expressions = { + level = "level || lvl || loglevel || LogLevel || log_level || logLevel || log_lvl || logLvl || levelname || levelName || LevelName", + } + } + + // set the extracted level as a label + stage.labels { + values = { + level = "", + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + // remove timestamp from the log line, depending on the entry it can be "start_time" or "time" + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + // unescaped regex: (?i)("(timestamp|ts|logdate|time)"\s*:\s*"[^"]+",?) + stage.replace { + expression = "(?i)(\"(timestamp|ts|logdate|time)\"\\s*:\\s*\"[^\"]+\",?)" + replace = "" + } + } + + // check logs.agent.grafana.com/scrub-level annotation, if true remove the level from the log line (it is still a label) + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_level=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-level: true" + + // remove level from the log line + stage.replace { + // unescaped regex: (?i)"(log)?(level|lvl)"\s*:\s*"[^"]+",? + expression = "(?i)(\"(log)?(level|lvl)\"\\s*:\\s*\"[^\"]+\",?)" + replace = "" + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/klog.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/klog.river new file mode 100644 index 00000000..f2238398 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/klog.river @@ -0,0 +1,106 @@ +/* +Module: log-format-klog +Description: Log Processing for klog (used by kube-state-metrics and more in kube-system) +Docs: https://github.com/kubernetes/klog +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_format_klog +} + +loki.process "log_format_klog" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains klog and the line matches the format, then process the line as + // a klog (https://github.com/kubernetes/klog) + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: klog" + // unescaped regex: ^[IWED]\d+\s+\d{2}:\d{2}:\d{2}\.\d+\s+\d+\s+\S+:\d+\]\s+.*$ + selector = "{log_type=\"\", logs_agent_grafana_com_log_format=~\"(?i).*(klog).*\"} |~ \"^[IWED]\\\\d+\\\\s+\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d+\\\\s+\\\\d+\\\\s+\\\\S+:\\\\d+\\\\]\\\\s+.*$\"" + + // set the log_type + stage.static_labels { + values = { + log_type = "klog", + } + } + + // extract log level, klog uses a single letter code for the level followed by the month and day i.e. I0119 + stage.regex { + expression = "((?P[A-Z])[0-9])" + } + + // extract log level, klog uses a single letter code for the level followed by the month and day i.e. I0119 + stage.regex { + expression = "((?P[A-Z])[0-9])" + } + + // if the extracted level is I set INFO + stage.replace { + source = "level" + expression = "(I)" + replace = "INFO" + } + + // if the extracted level is W set WARN + stage.replace { + source = "level" + expression = "(W)" + replace = "WARN" + } + + // if the extracted level is E set ERROR + stage.replace { + source = "level" + expression = "(E)" + replace = "ERROR" + } + + // if the extracted level is D set DEBUG + stage.replace { + source = "level" + expression = "(D)" + replace = "DEBUG" + } + + // set the extracted level to be a label + stage.labels { + values = { + level = "", + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + + // unescaped regex: ([0-9]{4}\s+[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+\s+) + stage.replace { + expression = "([0-9]{4}\\s+[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]+\\s+)" + replace = "" + } + } + + // check logs.agent.grafana.com/scrub-level annotation, if true remove the level from the log line (it is still a label) + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_level=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-level: true" + + // remove level from the log line + stage.replace { + // unescaped regex: (log)?(lvl|level)="?[^\s]+\s"? + expression = "(^(I|W|E|D))" + replace = "" + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/log4j.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/log4j.river new file mode 100644 index 00000000..3a534585 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/log4j.river @@ -0,0 +1,140 @@ +/* +Module: log-format-log4j +Description: Log Processing for Log4j +Docs: https://logging.apache.org/log4j/2.x/manual/layouts.html#json-template-layout + https://github.com/logstash/log4j-jsonevent-layout +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_format_log4j +} + +loki.process "log_format_log4j" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains log4j-json and the line matches the format, then process the line as log4j-json + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: log4j-json" + selector = "{logs_agent_grafana_com_log_format=~\"(?i).*(log4j-?json).*\"} |~ \"^\\\\s*\\\\{.+\\\\}\\\\s*$\"" + + // set the log_type + stage.static_labels { + values = { + log_type = "log4j", + } + } + + stage.json { + expressions = { + level = "level", + thread = "thread_name", + logger = "logger_name", + class = "class", + timestamp = "[\"@timestamp\"]", + } + } + + // set the extracted values as labels so they can be used by downstream components, most likely several labels + // will be dropped before being written to Loki + stage.labels { + values = { + level = "", + thread = "", + logger = "", + class = "", + timestamp = "", + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + // remove timestamp from the log line, depending on the entry it can be "start_time" or "time" + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + // unescaped regex: (?i)("@?timestamp"\s*:\s*"[^"]+",) + stage.replace { + expression = "(?i)(\"@?timestamp\"\\s*:\\s*\"[^\"]+\",)" + replace = "" + } + } + + // check logs.agent.grafana.com/scrub-level annotation, if true remove the level from the log line (it is still a label) + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_level=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-level: true" + + // remove level from the log line + stage.replace { + // unescaped regex: ("level"\s*:\s*"[^"]+",) + expression = "(\"level\"\\s*:\\s*\"[^\"]+\",)" + replace = "" + } + } + } + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains log4j-text and the line matches the format, then process the line as log4j-text + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: log4j-text" + // unescaped regex: ^\d{4}[^\[]+\[\S+\]\s+\w+\s+\S+\s+-\s+.*$ + selector = "{log_type=\"\", logs_agent_grafana_com_log_format=~\"(?i).*(log4j(-?te?xt)?).*\"} |~ \"^\\\\d{4}[^\\\\[]+\\\\[\\\\S+\\\\]\\\\s+\\\\w+\\\\s+\\\\S+\\\\s+-\\\\s+.*$\"" + + // set the log_type + stage.static_labels { + values = { + log_type = "log4j", + } + } + + // extract the timestamp, level, traceId, spanId, processId, thread, logger from the log line + stage.regex { + // unescaped regex: (?P[0-9]{4}-[0-9]{2}-[0-9]{2}(T|\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?(Z|(\+|-)[0-9]+)?)\s+(?P\w+)\s+\[(?P[^]]+)\] + expression = "(?P[0-9]{4}-[0-9]{2}-[0-9]{2}(T|\\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]+)?(Z|(\\+|-)[0-9]+)?)\\s+(?P\\w+)\\s+\\[(?P[^]]+)\\]" + } + + // set the extracted values as labels so they can be used by downstream components, most likely several labels + // will be dropped before being written to Loki + stage.labels { + values = { + level = "", + thread = "", + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + // remove timestamp from the log line, depending on the entry it can be "start_time" or "time" + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + // unescaped regex: ([0-9]{4}-[0-9]{2}-[0-9]{2}(T|\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?(Z|(\+|-)[0-9]+)?) + stage.replace { + expression = "([0-9]{4}-[0-9]{2}-[0-9]{2}(T|\\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]+)?(Z|(\\+|-)[0-9]+)?)" + replace = "" + } + } + + // check logs.agent.grafana.com/scrub-level annotation, if true remove the level from the log line (it is still a label) + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_level=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-level: true" + + // remove level from the log line + stage.replace { + // unescaped regex: (\[?(DEBUG|INFO|WARN|ERROR|FATAL|TRACE)\]\s*) + expression = "(\\[?(DEBUG|INFO|WARN|ERROR|FATAL|TRACE)\\]\\s*)" + replace = "" + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/logfmt.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/logfmt.river new file mode 100644 index 00000000..09c188e8 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/logfmt.river @@ -0,0 +1,73 @@ +/* +Module: log-format-logfmt +Description: Handles formatting for log format of logfmt which is the default Golang format +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_format_logfmt +} + +loki.process "log_format_logfmt" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains logfmt and the line matches the format, then process the line as + // a logfmt (https://github.com/go-logfmt/logfmt) + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: logfmt" + // unescaped regex: (\w+=("[^"]*"|\S+))(\s+(\w+=("[^"]*"|\S+)))*\s* + selector = "{log_type=\"\", logs_agent_grafana_com_log_format=~\"(?i).*(logfmt).*\"} |~ \"(\\\\w+=(\\\"[^\\\"]*\\\"|\\\\S+))(\\\\s+(\\\\w+=(\\\"[^\\\"]*\\\"|\\\\S+)))*\\\\s*\"" + + // set the log_type + stage.static_labels { + values = { + log_type = "logfmt", + } + } + + // while the level could be extracted as logfmt, this allows for multiple possible log levels formats + // i.e. loglevel=info, level=info, lvl=info, loglvl=info + stage.regex { + expression = "(log)?(level|lvl)=\"?(?P\\S+)\"?" + } + + // set the extracted level value as a label + stage.labels { + values = { + level = "", + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + + // unescaped regex: ((ts?|timestamp)=\d{4}-\d{2}-\d{2}(T|\s+)\d{2}:\d{2}:\d{2}(\.\d+)?(Z|(\+|-)\d+)?\s+) + stage.replace { + expression = "((ts?|timestamp)=[0-9]{4}-[0-9]{2}-[0-9]{2}(T|\\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]+)?(Z|(\\+|-)[0-9]+)?\\s+)" + replace = "" + } + } + + // check logs.agent.grafana.com/scrub-level annotation, if true remove the level from the log line (it is still a label) + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_level=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-level: true" + + // remove level from the log line + stage.replace { + // unescaped regex: (log)?(lvl|level)="?[^\s]+\s"? + expression = "(?i)((log)?(lvl|level)=\"?[^\\s]+\\s\"?)" + replace = "" + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/otel.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/otel.river new file mode 100644 index 00000000..6b2a50ac --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/otel.river @@ -0,0 +1,76 @@ +/* +Module: log-format-otel +Description: Log Processing for OpenTelemetry +Docs: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_format_otel +} + +loki.process "log_format_otel" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains otel and the line matches the format, then process the line as otel + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: otel" + selector = "{log_type=\"\", logs_agent_grafana_com_log_format=~\"(?i).*((otel|open-?telemetry)(-?json)).*\"} |~ \"^\\\\s*\\\\{.+\\\\}\\\\s*$\"" + + // set the log_type + stage.static_labels { + values = { + log_type = "otel", + } + } + + // extract the SeverityText (level), and service.name + // Docs: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/README.md#service + stage.json { + expressions = { + level = "SeverityText", + service = "Resource.\"service.name\"", + } + } + + // set the extracted level and service as labels + stage.labels { + values = { + level = "", + service = "", + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + // remove timestamp from the log line, depending on the entry it can be "start_time" or "time" + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + // unescaped regex: ("Timestamp"\s*:\s*"[^"]+",) + stage.replace { + expression = "(\"Timestamp\"\\s*:\\s*\"[^\"]+\",)" + replace = "" + } + } + + // check logs.agent.grafana.com/scrub-level annotation, if true remove the level from the log line (it is still a label) + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_level=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-level: true" + + // remove level from the log line + stage.replace { + // unescaped regex: ("SeverityText"\s*:\s*"[^"]+",) + expression = "(?i)(\"SeverityText\"\\s*:\\s*\"[^\"]+\",)" + replace = "" + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/postgres.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/postgres.river new file mode 100644 index 00000000..eae2f7e6 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/postgres.river @@ -0,0 +1,73 @@ +/* +Module: log-format-postgres +Description: Handles formatting for log format of Postgres +Docs: https://www.postgresql.org/docs/current/runtime-config-logging.html +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_format_postgres +} + +loki.process "log_format_postgres" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains postgres then process the line + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: postgres" + selector = "{log_type=\"\", logs_agent_grafana_com_log_format=~\"(?i).*(postgres).*\"}" + + // set the log_type + stage.static_labels { + values = { + log_type = "postgres", + } + } + + // extract the level and process_id from the log + // unescaped regex: \[?(?P\d{4}-\d{2}-\d{2}(T|\s+)\d{2}:\d{2}:\d{2}.\d+\s+\w+)\]?\s+(\[(?P\d+)\]\s+|.+)(?P(INFO|NOTICE|WARNING|ERROR|LOG|FATAL|PANIC|DEBUG)\d*):\s* + stage.regex { + expression = "\\[?(?P\\d{4}-\\d{2}-\\d{2}(T|\\s+)\\d{2}:\\d{2}:\\d{2}.\\d+\\s+\\w+)\\]?\\s+(\\[(?P\\d+)\\]\\s+|.+)(?P(INFO|NOTICE|WARNING|ERROR|LOG|FATAL|PANIC|DEBUG)\\d*):\\s*" + } + + // set the extracted level and process_id as labels + stage.labels { + values = { + level = "", + process_id = "", + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + + // unescaped regex: (\[?[0-9]{4}-[0-9]{2}-[0-9]{2}(T|\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]+\s+\w+\]?) + stage.replace { + expression = "(\\[?[0-9]{4}-[0-9]{2}-[0-9]{2}(T|\\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]+\\s+\\w+\\]?)" + replace = "" + } + } + + // check logs.agent.grafana.com/scrub-level annotation, if true remove the level from the log line (it is still a label) + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_level=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-level: true" + + // remove level from the log line + stage.replace { + // unescaped regex: ((INFO|NOTICE|WARNING|ERROR|LOG|FATAL|PANIC|DEBUG)\d*:\s+) + expression = "((INFO|NOTICE|WARNING|ERROR|LOG|FATAL|PANIC|DEBUG)\\d*:\\s+)" + replace = "" + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/python.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/python.river new file mode 100644 index 00000000..89b4e4a8 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/python.river @@ -0,0 +1,78 @@ +/* +Module: log-format-python +Description: Log Processing for Python +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_format_python +} + +loki.process "log_format_python" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains python-json and the line matches the format, then process the line as python-json + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: python-json" + selector = "{log_type=\"\", logs_agent_grafana_com_log_format=~\"(?i).*(python-?json).*\"} |~ \"^\\\\s*\\\\{.+\\\\}\\\\s*$\"" + + // set the log_type + stage.static_labels { + values = { + log_type = "python", + } + } + + // extract the level, response_code, method if they exist + stage.json { + expressions = { + level = "level || lvl || loglevel || log_level || logLevel || log_lvl || logLvl || levelname || levelName", + process = "processName || process_name || process", + module = "module || moduleName || module_name", + func = "funcName || func_name || func", + } + } + + // set the extracted level, process, module and func as labels + stage.labels { + values = { + level = "", + process = "", + module = "", + func = "", + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + // remove timestamp from the log line, depending on the entry it can be "start_time" or "time" + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + // unescaped regex: (?i)("(@?timestamp|asctime)"\s*:\s*\[?"[^"]+"\]?,?) + stage.replace { + expression = "(?i)(\"(@?timestamp|asctime)\"\\s*:\\s*\\[?\"[^\"]+\"\\]?,?)" + replace = "" + } + } + + // check logs.agent.grafana.com/scrub-level annotation, if true remove the level from the log line (it is still a label) + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_level=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-level: true" + + // remove level from the log line + stage.replace { + // unescaped regex: (?i)"(log)?(level|lvl)(name)?"\s*:\s*"[^"]+",? + expression = "(?i)(\"(log)?(level|lvl)(name)?\"\\s*:\\s*\"[^\"]+\",?)" + replace = "" + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/spring-boot.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/spring-boot.river new file mode 100644 index 00000000..a9a44b95 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/spring-boot.river @@ -0,0 +1,80 @@ +/* +Module: log-format-spring-boot +Description: Log Processing for Spring Boot +Docs: https://docs.spring.io/spring-boot/docs/2.1.13.RELEASE/reference/html/boot-features-logging.html +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_format_spring_boot +} + +loki.process "log_format_spring_boot" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains springboot and the line matches the format, then process the line as spring-boot + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: spring-boot" + // unescaped regex: ^\d{4}.+(INFO|ERROR|WARN|DEBUG|TRACE)\s+\d+\s+[^\[]+\[\S+\]\s+\S+\s+:\s+.*$ + selector = "{log_type=\"\", logs_agent_grafana_com_log_format=~\"(?i).*(spring-?boot).*\"} |~ \"^\\\\d{4}.+(INFO|ERROR|WARN|DEBUG|TRACE)\\\\s+\\\\d+\\\\s+[^\\\\[]+\\\\[\\\\S+\\\\]\\\\s+\\\\S+\\\\s+:\\\\s+.*$\"" + + // set the log_type + stage.static_labels { + values = { + log_type = "spring-boot", + } + } + + // extract the timestamp, level, traceId, spanId, processId, thread, logger from the log line + stage.regex { + // unescaped regex: (?P[0-9]{4}-[0-9]{2}-[0-9]{2}(T|\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?)\s+(?P\w+)\s+(?P\[(\S*\-?),(?P\S*),(?P\S*)\])\s+(?P[0-9]+)\s+-+\s+\[\s*(?P\S+)\]\s+(?P\S+)\s+:\s+(?P.+) + expression = "(?P[0-9]{4}-[0-9]{2}-[0-9]{2}(T|\\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]+)?)\\s+(?P\\w+)\\s+(?P\\[(\\S*\\-?),(?P\\S*),(?P\\S*)\\])\\s+(?P[0-9]+)\\s+-+\\s+\\[\\s*(?P\\S+)\\]\\s+(?P\\S+)\\s+:\\s+(?P.+)" + } + + // set the extracted values as labels so they can be used by downstream components, most likely several labels + // will be dropped before being written to Loki + stage.labels { + values = { + level = "", + trace = "", + traceId = "", + spanId = "", + processId = "", + thread = "", + logger = "", + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + // remove timestamp from the log line, depending on the entry it can be "start_time" or "time" + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + // unescaped regex: ^([0-9]{4}-[0-9]{2}-[0-9]{2}(T|\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?(Z|(\+|-)[0-9:]+)?)\s+ + stage.replace { + expression = "^([0-9]{4}-[0-9]{2}-[0-9]{2}(T|\\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]+)?(Z|(\\+|-)[0-9:]+)?)\\s+" + replace = "" + } + } + + // check logs.agent.grafana.com/scrub-level annotation, if true remove the level from the log line (it is still a label) + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_level=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-level: true" + + // remove level from the log line + stage.replace { + // unescaped regex: (ERROR|WARN|INFO|DEBUG|TRACE)\s+ + expression = "(ERROR|WARN|INFO|DEBUG|TRACE)\\s+" + replace = "" + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/syslog.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/syslog.river new file mode 100644 index 00000000..7a63f731 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/syslog.river @@ -0,0 +1,46 @@ +/* +Module: log-format-syslog +Description: Handles formatting for log format of syslog +Docs: https://datatracker.ietf.org/doc/html/rfc5424 +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_format_syslog +} + +loki.process "log_format_syslog" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains postgres then process the line + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: syslog" + // unescaped regex: ^[A-Za-z]{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\s+\S+\s+\S+\[\d+\]:\s+.*$ + selector = "{log_type=\"\", logs_agent_grafana_com_log_format=~\"(?i).*(syslog).*\"} |~ \"^[A-Za-z]{3}\\\\s+\\\\d{1,2}\\\\s+\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\s+\\\\S+\\\\s+\\\\S+\\\\[\\\\d+\\\\]:\\\\s+.*$\"" + + stage.static_labels { + values = { + // set the log_type + log_type = "syslog", + level = "info", + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + // unescaped regex: ^[A-Za-z]{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2} + stage.replace { + expression = "(^[A-Za-z]{3}\\s+\\d{1,2}\\s+\\d{2}:\\d{2}:\\d{2})" + replace = "" + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/zerolog.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/zerolog.river new file mode 100644 index 00000000..5c1c097d --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/log-formats/zerolog.river @@ -0,0 +1,122 @@ +/* +Module: log-format-zerolog +Description: Handles formatting for log format of zerolog +Docs: https://github.com/rs/zerolog +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.log_format_zerolog +} + +loki.process "log_format_zerolog" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/log-format annotation, if the log_type is empty the line hasn't been processed, if it contains postgres then process the line + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/log-format: zerolog" + // unescaped regex: ^.+(TRC|DBG|INF|WRN|ERR|FTL|PNC)[^=]+(\w+=("[^"]*"|\S+))(\s+(\w+=("[^"]*"|\S+)))*\s*$ + selector = "{log_type=\"\", logs_agent_grafana_com_log_format=~\"(?i).*(zero-?log).*\"} |~ \"^.+(TRC|DBG|INF|WRN|ERR|FTL|PNC)[^=]+(\\\\w+=(\\\"[^\\\"]*\\\"|\\\\S+))(\\\\s+(\\\\w+=(\\\"[^\\\"]*\\\"|\\\\S+)))*\\\\s*$\"" + + // set the log_type + stage.static_labels { + values = { + log_type = "zerolog", + } + } + + // extract the level from the log + // unescaped regex: (?P[0-9]{4}-[0-9]{2}-[0-9]{2}(T|\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]+[^ ]*\s+)(?P(TRC|DBG|INF|WRN|ERR|FTL|PNC)).+ + stage.regex { + expression = "(?P[0-9]{4}-[0-9]{2}-[0-9]{2}(T|\\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]+[^ ]*\\s+)(?P(TRC|DBG|INF|WRN|ERR|FTL|PNC)).+" + } + + // if the extracted level is TRC set trace + stage.replace { + source = "level" + expression = "(TRC)" + replace = "trace" + } + + // if the extracted level is DBG set debug + stage.replace { + source = "level" + expression = "(DBG)" + replace = "debug" + } + + // if the extracted level is INF set info + stage.replace { + source = "level" + expression = "(INF)" + replace = "info" + } + + // if the extracted level is WRN set warn + stage.replace { + source = "level" + expression = "(WRN)" + replace = "warn" + } + + // if the extracted level is ERR set error + stage.replace { + source = "level" + expression = "(ERR)" + replace = "error" + } + + // if the extracted level is FTL set fatal + stage.replace { + source = "level" + expression = "(FTL)" + replace = "fatal" + } + + // if the extracted level is PNC set panic + stage.replace { + source = "level" + expression = "(PNC)" + replace = "panic" + } + + // set the extracted level as a labels + stage.labels { + values = { + level = "", + } + } + + // check logs.agent.grafana.com/scrub-timestamp annotation, if true remove the timestamp from the log line + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_timestamp=\"true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-timestamp: true" + + // remove timestamp from the log line + + // unescaped regex: ([0-9]{4}-[0-9]{2}-[0-9]{2}(T|\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]+[^ ]*\s+) + stage.replace { + expression = "([0-9]{4}-[0-9]{2}-[0-9]{2}(T|\\s+)[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]+[^ ]*\\s+)" + replace = "" + } + } + + // check logs.agent.grafana.com/scrub-level annotation, if true remove the level from the log line (it is still a label) + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_level=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-level: true" + + // remove level from the log line + stage.replace { + // unescaped regex: (TRC|DBG|INF|WRN|ERR|FTL|PNC)\s+ + expression = "(TRC|DBG|INF|WRN|ERR|FTL|PNC)\\s+" + replace = "" + } + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/all.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/all.river new file mode 100644 index 00000000..1e7c5563 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/all.river @@ -0,0 +1,93 @@ +/* +Module: mask-all +Description: Wrapper module to include all masking modules +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +argument "git_repo" { + optional = true + default = coalesce(env("GIT_REPO"), "https://github.com/grafana/agent-modules.git") +} + +argument "git_rev" { + optional = true + default = coalesce(env("GIT_REV"), env("GIT_REVISION"), env("GIT_BRANCH"), "main") +} + +argument "git_pull_freq" { + optional = true + default = "5m" +} + +export "process" { + value = module.git.mask_ssn.exports.process +} + +module.git "mask_ssn" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/masks/ssn.river" + + arguments { + forward_to = [module.git.mask_credit_card.exports.process.receiver] + } +} + +module.git "mask_credit_card" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/masks/credit-card.river" + + arguments { + forward_to = [module.git.mask_email.exports.process.receiver] + } +} + +module.git "mask_email" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/masks/email.river" + + arguments { + forward_to = [module.git.mask_phone.exports.process.receiver] + } +} + +module.git "mask_phone" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/masks/phone.river" + + arguments { + forward_to = [module.git.mask_ipv4.exports.process.receiver] + } +} + +module.git "mask_ipv4" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/masks/ipv4.river" + + arguments { + forward_to = [module.git.mask_ipv6.exports.process.receiver] + } +} + +module.git "mask_ipv6" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/masks/ipv6.river" + + arguments { + forward_to = argument.forward_to.value + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/credit-card.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/credit-card.river new file mode 100644 index 00000000..e0f4c8f5 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/credit-card.river @@ -0,0 +1,35 @@ +/* +Module: mask-credit-card +Description: Checks the logs.agent.grafana.com/mask-credit-card annotation, if set to "true" any logs that match the credit + card pattern will have the value of the credit card replaced with "*credit-card*hash* +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.mask_credit_card +} + +loki.process "mask_credit_card" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/mask-credit-card annotation, if true the data will be masked as *credit-card*salt* + // Formats: + // Visa: 4[0-9]{15} + // MasterCard: 5[1-5][0-9]{14} + // American Express: 3[47][0-9]{13} + // Discover: 6[0-9]{15} + // JCB: 3[51-55][0-9]{14} + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/mask-credit-card: true" + selector = "{logs_agent_grafana_com_mask_credit_card=~\"(?i)true\"}" + + stage.replace { + // unescaped regex: (4[0-9]{15}|5[1-5][0-9]{14}|3[47][0-9]{13}|6[0-9]{15}|3[51-55][0-9]{14}) + expression = "(4[0-9]{15}|5[1-5][0-9]{14}|3[47][0-9]{13}|6[0-9]{15}|3[51-55][0-9]{14})" + replace = "*credit-card*{{ .Value | Hash \"salt\" }}*" + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/email.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/email.river new file mode 100644 index 00000000..bd047e36 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/email.river @@ -0,0 +1,29 @@ +/* +Module: mask-email +Description: Checks the logs.agent.grafana.com/mask-email annotation, if set to "true" any logs that match the email + pattern will have the value of the email replaced with "*email*hash* +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.mask_email +} + +loki.process "mask_email" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/mask-email annotation, if true the data will be masked as *email*salt* + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/mask-email: true" + selector = "{logs_agent_grafana_com_mask_email=~\"(?i)true\"}" + + stage.replace { + // unescaped regex: ([\w\.=-]+@[\w\.-]+\.[\w]{2,64}) + expression = "([\\w\\.=-]+@[\\w\\.-]+\\.[\\w]{2,64})" + replace = "*email*{{ .Value | Hash \"salt\" }}*" + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/ipv4.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/ipv4.river new file mode 100644 index 00000000..bfd5b35c --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/ipv4.river @@ -0,0 +1,29 @@ +/* +Module: mask-ipv4 +Description: Checks the logs.agent.grafana.com/mask-ipv4 annotation, if set to "true" any logs that match the ipv4 + pattern will have the value of the ipv4 replaced with "*ipv4*hash* +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.mask_ipv4 +} + +loki.process "mask_ipv4" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/mask-ipv4 annotation, if true the data will be masked as *ipv4*salt* + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/mask-ipv4: true" + selector = "{logs_agent_grafana_com_mask_ipv4=~\"(?i)true\"}" + + stage.replace { + // unescaped regex: ((\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}) + expression = "((\\b25[0-5]|\\b2[0-4][0-9]|\\b[01]?[0-9][0-9]?)(\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3})" + replace = "*ipv4*{{ .Value | Hash \"salt\" }}*" + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/ipv6.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/ipv6.river new file mode 100644 index 00000000..d02f8c37 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/ipv6.river @@ -0,0 +1,29 @@ +/* +Module: mask-ipv6 +Description: Checks the logs.agent.grafana.com/mask-ipv6 annotation, if set to "true" any logs that match the ipv6 + pattern will have the value of the ipv6 replaced with "*ipv6*hash* +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.mask_ipv6 +} + +loki.process "mask_ipv6" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/mask-ipv6 annotation, if true the data will be masked as *ipv6*salt* + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/mask-ipv6: true" + selector = "{logs_agent_grafana_com_mask_ipv6=~\"(?i)true\"}" + + stage.replace { + // unescaped regex: (([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])) + expression = "(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))" + replace = "*ipv6*{{ .Value | Hash \"salt\" }}*" + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/phone.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/phone.river new file mode 100644 index 00000000..0152c21b --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/phone.river @@ -0,0 +1,29 @@ +/* +Module: mask-phone +Description: Checks the logs.agent.grafana.com/mask-phone annotation, if set to "true" any logs that match the phone + pattern will have the value of the phone replaced with "*phone*hash* +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.mask_phone +} + +loki.process "mask_phone" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/mask-phone annotation, if true the data will be masked as *phone*salt* + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/mask-phone: true" + selector = "{logs_agent_grafana_com_mask_phone=~\"(?i)true\"}" + + stage.replace { + // unescaped regex: ([\+]?[(]?[0-9]{3}[)]?[-\s\.]?[0-9]{3}[-\s\.]?[0-9]{4,6}) + expression = "([\\+]?[(]?[0-9]{3}[)]?[-\\s\\.]?[0-9]{3}[-\\s\\.]?[0-9]{4,6})" + replace = "*phone*{{ .Value | Hash \"salt\" }}*" + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/ssn.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/ssn.river new file mode 100644 index 00000000..f0fd58ad --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/masks/ssn.river @@ -0,0 +1,29 @@ +/* +Module: mask-ssn +Description: Checks the logs.agent.grafana.com/mask-ssn annotation, if set to "true" any logs that match the ssn + pattern will have the value of the ssn replaced with "*ssn*hash* +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.mask_ssn +} + +loki.process "mask_ssn" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/mask-ssn annotation, if true the data will be masked as *ssn*salt* + stage.match { + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/mask-ssn: true" + selector = "{logs_agent_grafana_com_mask_ssn=~\"(?i)true\"}" + + stage.replace { + // unescaped regex: ([0-9]{3}-[0-9]{2}-[0-9]{4}) + expression = "([0-9]{3}-[0-9]{2}-[0-9]{4})" + replace = "*ssn*{{ .Value | Hash \"salt\" }}*" + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/metrics/post-process-bytes-lines.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/metrics/post-process-bytes-lines.river new file mode 100644 index 00000000..5464af9c --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/metrics/post-process-bytes-lines.river @@ -0,0 +1,39 @@ +/* +Module: pre-process-lines-bytes-metrics +Description: Generates metrics for the number of lines and bytes in the log line before any processing is done +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.pre_process_lines_bytes_metrics +} + +loki.process "pre_process_lines_bytes_metrics" { + forward_to = argument.forward_to.value + + stage.metrics { + metric.counter { + name = "lines_total" + description = "total number of log lines ingested, processed and forwarded for storage" + prefix = "log_" + match_all = true + action = "inc" + max_idle_duration = "24h" + } + } + + stage.metrics { + metric.counter { + name = "bytes_total" + description = "total log bytes ingested, processed and forwarded for storage" + prefix = "log_" + match_all = true + count_entry_bytes = true + action = "add" + max_idle_duration = "24h" + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/metrics/pre-process-bytes-lines.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/metrics/pre-process-bytes-lines.river new file mode 100644 index 00000000..972cdbe8 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/metrics/pre-process-bytes-lines.river @@ -0,0 +1,93 @@ +/* +Module: pre-process-lines-bytes-metrics +Description: Generates metrics for the number of lines and bytes in the log line before any processing is done +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +argument "keep_labels" { + optional = true + // comment = "List of labels to keep before the log message is written to Loki" + default = [ + "app", + "cluster", + "component", + "container", + "deployment", + "env", + "filename", + "instance", + "job", + "level", + "log_type", + "namespace", + "region", + "service", + "squad", + "team", + ] +} + +argument "git_repo" { + optional = true + default = coalesce(env("GIT_REPO"), "https://github.com/grafana/agent-modules.git") +} + +argument "git_rev" { + optional = true + default = coalesce(env("GIT_REV"), env("GIT_REVISION"), env("GIT_BRANCH"), "main") +} + +argument "git_pull_freq" { + // comment = "How often to pull the git repo, the default is 0s which means never pull" + optional = true + default = "0s" +} + +export "process" { + value = module.git.label_keep.exports.process +} + +// drop any labels that are not in the keep_labels list +// this is because the metrics generated below will keep the full set of labels currently attached to the log line +// we want those to line up with what we're keeping +module.git "label_keep" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/labels/keep-labels.river" + + arguments { + forward_to = [loki.process.pre_process_lines_bytes_metrics.receiver] + keep_labels = argument.keep_labels.value + } +} + +loki.process "pre_process_lines_bytes_metrics" { + forward_to = [] // does not forward anywhere, just generates metrics + + stage.metrics { + metric.counter { + name = "lines_pre_total" + description = "total number of log lines ingested before processing" + prefix = "log_" + match_all = true + action = "inc" + max_idle_duration = "24h" + } + } + + stage.metrics { + metric.counter { + name = "bytes_pre_total" + description = "total number of log bytes ingested before processing" + prefix = "log_" + match_all = true + count_entry_bytes = true + action = "add" + max_idle_duration = "24h" + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/relabelings.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/relabelings.river new file mode 100644 index 00000000..f7d63ebf --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/relabelings.river @@ -0,0 +1,142 @@ +/* +Module: log-relabelings +Description: Handles log relabelings +*/ +argument "targets" { + // comment = "Discovered targets to apply relabelings to" + optional = false +} + +argument "tenant" { + // comment = "The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this can be a regex." + optional = true + default = "^(primary|)$" +} + +argument "git_repo" { + optional = true + default = coalesce(env("GIT_REPO"), "https://github.com/grafana/agent-modules.git") +} + +argument "git_rev" { + optional = true + default = coalesce(env("GIT_REV"), env("GIT_REVISION"), env("GIT_BRANCH"), "main") +} + +argument "git_pull_freq" { + optional = true + default = "5m" +} + +export "relabelings" { + value = discovery.relabel.logs +} + +// apply common relabelings +module.git "relabelings_common" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/relabelings/common.river" + + arguments { + targets = argument.targets.value + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +// apply common pod relabelings +module.git "relabelings_pod" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/relabelings/pod.river" + + arguments { + targets = module.git.relabelings_common.exports.relabelings.output + } +} + +// apply pod log specific relabelings +discovery.relabel "logs" { + targets = module.git.relabelings_pod.exports.relabelings.output + + // allow pods to declare their logs to be ingested or not, the following annotation is supported: + // logs.agent.grafana.com/ingest: false + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_annotation_logs_agent_grafana_com_ingest"] + separator = ";" + regex = "^(?:;*)?(true|false).*$" + replacement = "$1" + target_label = "__tmp_ingest" + } + + // drop any targets that have ingest: false + rule { + action = "drop" + source_labels = ["__tmp_ingest"] + regex = "false" + } + + // allow pods to declare what tenant their logs should be written to, the following annotation is supported: + // logs.agent.grafana.com/tenant: "primary" + rule { + action = "keep" + source_labels = ["__meta_kubernetes_pod_annotation_logs_agent_grafana_com_tenant"] + regex = "^(" + argument.tenant.value + ")$" + } + + // set the __path__, this is automatically translated as a label of filename (which should be dropped or normalized) + // DO NOT delete this line as it is needed to tail the pod logs on the node + rule { + action = "replace" + separator = "/" + source_labels = [ + "__meta_kubernetes_pod_uid", + "__meta_kubernetes_pod_container_name", + ] + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + // set the __host__ + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "__host__" + } + + // set the job label to be namespace / friendly pod name + rule { + action = "replace" + source_labels = [ + "deployment", + "__meta_kubernetes_namespace", + ] + regex = ".+\\/(.+);(.+)" + replacement = "$2/$1" + target_label = "job" + } + + // set the container label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + + // make all labels on the pod available to the pipeline as labels, + // they are omitted before write via labelallow unless explicitly set + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_label_(.+)" + } + // make all annotations on the pod available to the pipeline as labels, + // they are omitted before write via labelallow unless explicitly set + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_annotation_(.+)" + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/scrubs/all.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/scrubs/all.river new file mode 100644 index 00000000..13fb2188 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/scrubs/all.river @@ -0,0 +1,49 @@ +/* +Module: scrub-all +Description: Wrapper module to include all scrubing modules +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +argument "git_repo" { + optional = true + default = coalesce(env("GIT_REPO"), "https://github.com/grafana/agent-modules.git") +} + +argument "git_rev" { + optional = true + default = coalesce(env("GIT_REV"), env("GIT_REVISION"), env("GIT_BRANCH"), "main") +} + +argument "git_pull_freq" { + optional = true + default = "5m" +} + +export "process" { + value = module.git.scrub_json_empties.exports.process +} + +module.git "scrub_json_empties" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/scrubs/json-empties.river" + + arguments { + forward_to = [module.git.scrub_json_nulls.exports.process.receiver] + } +} + +module.git "scrub_json_nulls" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/scrubs/json-nulls.river" + + arguments { + forward_to = argument.forward_to.value + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/scrubs/json-empties.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/scrubs/json-empties.river new file mode 100644 index 00000000..0e38270a --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/scrubs/json-empties.river @@ -0,0 +1,32 @@ +/* +Module: scrub-json-empties +Description: Checks for the annotation logs.agent.grafana.com/scrub-empties, if set to "true" + Removes any json properties with empty values i.e. "foo": "", "bar": [], "baz": {} +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.scrub_json_empties +} + +loki.process "scrub_json_empties" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/scrub-empties annotation, if true remove any json property whose value is set to + // an empty string "", empty object {} or empty array [] is removed + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_empties=~\"(?i)(dotnet-?json|istio|(generic-?)?json|log4j-?json|(otel|open-?telemetry)(-?json)?|python-?json)\",logs_agent_grafana_com_scrub_nulls=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-null: true" + + // remove null properties + stage.replace { + // unescaped regex: (\s*,\s*("[^"]+"\s*:\s*(\[\s*\]|\{\s*\}|"\s*"))|("[^"]+"\s*:\s*(\[\s*\]|\{\s*\}|"\s*"))\s*,\s*) + expression = "(\\s*,\\s*(\"[^\"]+\"\\s*:\\s*(\\[\\s*\\]|\\{\\s*\\}|\"\\s*\"))|(\"[^\"]+\"\\s*:\\s*(\\[\\s*\\]|\\{\\s*\\}|\"\\s*\"))\\s*,\\s*)" + replace = "" + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/scrubs/json-nulls.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/scrubs/json-nulls.river new file mode 100644 index 00000000..1a6349fa --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/scrubs/json-nulls.river @@ -0,0 +1,31 @@ +/* +Module: scrub-json-nulls +Description: Checks for the annotation logs.agent.grafana.com/scrub-nulls, if set to "true" + Removes any json properties with a null value +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +export "process" { + value = loki.process.scrub_json_nulls +} + +loki.process "scrub_json_nulls" { + forward_to = argument.forward_to.value + + // check logs.agent.grafana.com/scrub-nulls annotation, if true remove any json property whose value is set to null + // this can reduce the overall # of bytes sent and stored in Loki + stage.match { + selector = "{logs_agent_grafana_com_scrub_nulls=~\"(?i)(dotnet-?json|istio|(generic-?)?json|log4j-?json|(otel|open-?telemetry)(-?json)?|python-?json)\",logs_agent_grafana_com_scrub_nulls=~\"(?i)true\"}" + pipeline_name = "pipeline for annotation || logs.agent.grafana.com/scrub-null: true" + + // remove null properties + stage.replace { + // unescaped regex: (\s*,\s*("[^"]+"\s*:\s*null)|("[^"]+"\s*:\s*null)\s*,\s*) + expression = "(\\s*,\\s*(\"[^\"]+\"\\s*:\\s*null)|(\"[^\"]+\"\\s*:\\s*null)\\s*,\\s*)" + replace = "" + } + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/targets/logs-from-api.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/targets/logs-from-api.river new file mode 100644 index 00000000..eee66f82 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/targets/logs-from-api.river @@ -0,0 +1,48 @@ +/* +Module: logs-from-api +Description: Performs Kubernetes service discovery for pods, applies relabelings, the discovered target logs are + then retrieved from the kubernetes api +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +argument "git_repo" { + optional = true + default = coalesce(env("GIT_REPO"), "https://github.com/grafana/agent-modules.git") +} + +argument "git_rev" { + optional = true + default = coalesce(env("GIT_REV"), env("GIT_REVISION"), env("GIT_BRANCH"), "main") +} + +argument "git_pull_freq" { + optional = true + default = "5m" +} + +discovery.kubernetes "pods" { + role = "pod" +} + +module.git "relabelings_log" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/relabelings.river" + + arguments { + targets = discovery.kubernetes.pods.targets + tenant = argument.tenant.value + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +loki.source.kubernetes "pods" { + targets = module.git.relabelings_log.exports.relabelings.output + forward_to = argument.forward_to.value +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/targets/logs-from-worker.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/targets/logs-from-worker.river new file mode 100644 index 00000000..1e1314ba --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/logs/targets/logs-from-worker.river @@ -0,0 +1,118 @@ +/* +Module: logs-from-worker +Description: Performs Kubernetes service discovery for pods, applies relabelings, discovers available files on the + worker node, and uses these are source files for loki and parses the files using the cri stage. + + This requires the agent to deployed as a DaemonSet, as each pod will mount a volume to the worker + node to retrieve the pod logs. +*/ +argument "forward_to" { + // comment = "Must be a list(LogsReceiver) where collected logs should be forwarded to" + optional = false +} + +argument "tenant" { + // comment = "The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this can be a regex." + optional = true + default = ".*" +} + +argument "git_repo" { + optional = true + default = coalesce(env("GIT_REPO"), "https://github.com/grafana/agent-modules.git") +} + +argument "git_rev" { + optional = true + default = coalesce(env("GIT_REV"), env("GIT_REVISION"), env("GIT_BRANCH"), "main") +} + +argument "git_pull_freq" { + optional = true + default = "5m" +} + +discovery.kubernetes "pods" { + role = "pod" +} + +module.git "relabelings_log" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/logs/relabelings.river" + + arguments { + targets = discovery.kubernetes.pods.targets + tenant = argument.tenant.value + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +// as a result of kubernetes service discovery for pods, all of the meta data information is exposed in labels +// __meta_kubernetes_pod_*, including __meta_kubernetes_pod_container_id which can be used to determine what +// the pods container runtime is, docker (docker://...) or containerd (containerd://...) this will inform us +// which parsing stage to use. However, any labels that begin with __* are not passed to loki.process +// (pipeline) stages. Use a relabeling stage to set a label that can be used a LogQL selector in the stage +// below so parsing can be automatically determined, then drop the label from the loki.process stage. +discovery.relabel "container_runtime" { + targets = module.git.relabelings_log.exports.relabelings.output + + // set the container runtime as a label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = "^(\\w+):\\/\\/.+$" + replacement = "$1" + target_label = "tmp_container_runtime" + } +} + +local.file_match "pods" { + path_targets = discovery.relabel.container_runtime.output +} + +loki.source.file "pods" { + targets = local.file_match.pods.targets + forward_to = [loki.process.parse.receiver] +} + +loki.process "parse" { + forward_to = argument.forward_to.value + + // if the label tmp_container_runtime from above is containerd parse using cri + stage.match { + selector = "{tmp_container_runtime=\"containerd\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri { } + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + // if the label tmp_container_runtime from above is docker parse using docker + stage.match { + selector = "{tmp_container_runtime=\"docker\"}" + // the docker processing stage extracts the following k/v pairs: log, stream, time + stage.docker { } + + // Set the extract stream value as a label + stage.labels { + values = { + stream = "", + } + } + } + + // drop the temporary container runtime label as it is no longer needed + stage.label_drop { + values = ["tmp_container_runtime"] + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/all.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/all.river new file mode 100644 index 00000000..754ef0e1 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/all.river @@ -0,0 +1,172 @@ +/* +Module: metrics-all +Description: Wrapper module to include all kubernetes metric modules and use cri parsing +*/ +argument "forward_to" { + // comment = "Must be a list(MetricssReceiver) where collected logs should be forwarded to" + optional = false +} + +argument "scrape_port_named_metrics" { + // comment = "Whether or not to automatically scrape endpoints that have a port with 'metrics' in the name" + optional = true + default = false +} + +argument "clustering" { + // comment = "Whether or not clustering should be enabled" + optional = true + default = false +} + +argument "blackbox_url" { + // comment = "The address of the blackbox exporter to use (without the protocol), only the hostname and port i.e. blackbox-prometheus-blackbox-exporter.default.svc.cluster.local:9115" + optional = true + default = "" +} + +argument "drop_metrics" { + optional = true + // blackbox does not return that many metrics, however if certain metrics should be dropped they can be specified here + // the default is "none" which will not match any of the returned metric names + default = "probe_ip_addr_hash" + // comment = "Regex of metrics to drop" +} + +argument "tenant" { + // comment = "The tenant to filter logs to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this can be a regex." + optional = true + default = ".*" +} + +argument "git_repo" { + optional = true + default = coalesce(env("GIT_REPO"), "https://github.com/grafana/agent-modules.git") +} + +argument "git_rev" { + optional = true + default = coalesce(env("GIT_REV"), env("GIT_REVISION"), env("GIT_BRANCH"), "main") +} + +argument "git_pull_freq" { + optional = true + default = "0s" +} + +module.git "scrape_kubelet_cadvisor" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/metrics/scrapes/kubelet-cadvisor.river" + + arguments { + forward_to = argument.forward_to.value + tenant = argument.tenant.value + clustering = argument.clustering.value + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +module.git "scrape_kubelet" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/metrics/scrapes/kubelet.river" + + arguments { + forward_to = argument.forward_to.value + tenant = argument.tenant.value + clustering = argument.clustering.value + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +module.git "scrape_kubelet_probes" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/metrics/scrapes/kubelet-probes.river" + + arguments { + forward_to = argument.forward_to.value + tenant = argument.tenant.value + clustering = argument.clustering.value + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +module.git "scrape_kube_apiserver" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/metrics/scrapes/kube-apiserver.river" + + arguments { + forward_to = argument.forward_to.value + tenant = argument.tenant.value + clustering = argument.clustering.value + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +module.git "scrape_endpoints" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/metrics/scrapes/auto-scrape-endpoints.river" + + arguments { + forward_to = argument.forward_to.value + tenant = argument.tenant.value + clustering = argument.clustering.value + scrape_port_named_metrics = argument.scrape_port_named_metrics.value + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +module.git "probe_services" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/metrics/scrapes/auto-probe-services.river" + + arguments { + forward_to = argument.forward_to.value + tenant = argument.tenant.value + clustering = argument.clustering.value + blackbox_url = argument.blackbox_url.value + drop_metrics = argument.drop_metrics.value + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} + +module.git "probe_ingresses" { + repository = argument.git_repo.value + revision = argument.git_rev.value + pull_frequency = argument.git_pull_freq.value + path = "modules/kubernetes/metrics/scrapes/auto-probe-ingresses.river" + + arguments { + forward_to = argument.forward_to.value + tenant = argument.tenant.value + clustering = argument.clustering.value + blackbox_url = argument.blackbox_url.value + drop_metrics = argument.drop_metrics.value + git_repo = argument.git_repo.value + git_rev = argument.git_rev.value + git_pull_freq = argument.git_pull_freq.value + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/README.md b/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/README.md new file mode 100644 index 00000000..20a38855 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/README.md @@ -0,0 +1,525 @@ +# Kubernetes Metrics Jobs + +The following jobs are completely isolated and have no dependencies on other modules. Overall, they should be more performant than relying on dependencies of other modules as they are designed to complete a specific task. + +## Job Modules + +- [Agent](#agent) +- [Annotations Probe](#annotations-probe) +- [Annotations Scrape](#annotation-scrape) +- [cAdvisor](#cadvisor) +- [Cert Manager](#cert-manager) +- [Consul](#consul) +- [Etcd](#etcd) +- [Gitlab Exporter](#gitlab-exporter) +- [Grafana](#grafana) +- [HAProxy](#haproxy) +- [Kube ApiServer](#kube-apiserver) +- [Kube Probes](#kube-probes) +- [Kube Proxy](#kube-proxy) +- [Kube Resource](#kube-resource) +- [Kube State Metrics](#kube-state-metrics) +- [Kubelet](#kubelet) +- [Loki](#loki) +- [Memcached](#memcached) +- [Mimir](#mimir) +- [Mysql](#mysql) +- [Node Exporter](#node-exporter) +- [OpenCost](#opencost) +- [Prometheus Operator](#prometheus-operator) +- [Push Gateway](#push-gateway) +- [RabbitMQ](#rabbitmq) +- [Redis](#redis) +- [Statsd](#Statsd) +- [Tempo](#tempo) + +### Agent + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the grafana-agent job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=grafana-agent"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `http-metrics` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/agent` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./agent.river#L170) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Annotations Probe + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `role` | `false` | `service` | The role to use when looking for targets to scrape via annotations, can be: service or ingress | +| `blackbox_url` | `false` | `""` | The address of the blackbox exporter to use (without the protocol), only the hostname and port i.e. blackbox-prometheus-blackbox-exporter.default.svc.cluster.local:9115 | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `[]` | The label selectors to use to find matching targets | +| `annotation` | `false` | `probes.grafana.com` | The annotation namespace to use | +| `tenant` | `false` | `.*` | The tenant to write metrics to. This does not have to be the tenantId, this is the value to look for in the pro0bes.grafana.com/tenant annotation, and this can be a regex. | +| `keep_metrics` | `false` | `(.+)` | A regex of metrics to keep | +| `drop_metrics` | `false` | `""` | A regex of metrics to drop | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Annotations Scrape + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `role` | `false` | `service` | The role to use when looking for targets to scrape via annotations, can be: service or ingress | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `[]` | The label selectors to use to find matching targets | +| `annotation` | `false` | `probes.grafana.com` | The annotation namespace to use | +| `tenant` | `false` | `.*` | The tenant to write metrics to. This does not have to be the tenantId, this is the value to look for in the pro0bes.grafana.com/tenant annotation, and this can be a regex. | +| `keep_metrics` | `false` | `(.+)` | A regex of metrics to keep | +| `drop_metrics` | `false` | `""` | A regex of metrics to drop | +| `scrape_port_named_metrics` | `false` | Whether or not to automatically scrape endpoints that have a port with 'metrics' in the name | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### cAdvisor + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the cAdvisor job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `job_label` | `false` | `integrations/kubernetes/cadvisor` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./cadvisor.river#L115) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Cert Manager + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the cert-manager should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=cert-manager"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `http-metrics` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/cert-manager` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./cert-manager.river#L183) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Consul + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the consul should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app=consul"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `http-metrics` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/consul` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./consul.river#L183) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Etcd + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the etcd should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/component=etcd"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `metrics` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/consul` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./etcd.river#L183) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Gitlab Exporter + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the gitlab-exporter should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=gitlab-ci-pipelines-exporter"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `http` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/gitlab` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./gitlab.river#L128) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Grafana + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the grafana should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=grafana"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `http-metrics` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/gitlab` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./agent.river#L128) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### HAProxy + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the haproxy should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/component=haproxy"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `prometheus` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/haproxy` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./haproxy.river#L183) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Kube ApiServer + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the Kube ApiServer job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespace` | `false` | `default` | The namespaces to look for targets in | +| `service` | `false` | `kubernetes` | The label to use for the selector | +| `port_name` | `false` | `https` | The value of the label for the selector | +| `job_label` | `false` | `integrations/kubernetes/kube-apiserver` | The job label to add for all kube-apiserver metric | +| `drop_metrics` | `false` | [see module](./kube-apiserver.river#L153) | A regex of metrics to drop | +| `drop_les` | `false` | [see module](./kube-apiserver.river#163) | Regex of metric les label values to drop | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Kube Probes + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the Kube Probes job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `job_label` | `false` | `integrations/kubernetes/kube-probes` | The job label to add for all kube-probe metric | +| `keep_metrics` | `false` | [see module](./kube-probes.river#L117) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Kube Proxy + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the Kube Proxy job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespace` | `false` | `kube-system` | The namespaces to look for targets in | +| `selectors` | `false` | `["component=kube-proxy"]` | The label selectors to use to find matching targets | +| `port` | `false` | `10249` | The port to scrape kube-proxy metrics on | +| `job_label` | `false` | `integrations/kubernetes/kube-proxy` | The job label to add for all kube-apiserver metric | +| `keep_metrics` | `false` | [see module](./kube-proxy.river#L124) | A regex of metrics to drop | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Kube Resource + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the Kube Resources job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `job_label` | `false` | `integrations/kubernetes/kube-resources` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./kube-resource.river#L117) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Kube State Metrics + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the kube-state-metrics job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=kube-state-metrics"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `http-metrics` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/kubenetes/kube-state-metrics` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./kube-state-metrics.river#L127) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Kubelet + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the Kubelet job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `job_label` | `false` | `integrations/kubernetes/kubelet` | The job label to add for all kube-probe metric | +| `keep_metrics` | `false` | [see module](./kubelet.river#L116) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Loki + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the Loki job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=loki"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `http-metrics` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/loki` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./loki.river#L186) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Memcached + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the Memcached job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=memcached"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `metrics` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/memcached` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./memcached.river#L170) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Mimir + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the Mimir job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=mimir"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `http-metrics` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/mimir` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./mimir.river#L186) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Mysql + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the haproxy should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=prometheus-mysql-exporter"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `metrics` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/mysql` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./mysql.river#L183) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Node Exporter + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the Node Exporter job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=prometheus-node-exporter"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `metrics` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/node_exporter` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./node-exporter.river#L141) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### OpenCost + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the OpenCost job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=opencost"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `http` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/kubernetes/opencost` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./opencost.river#L123) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Prometheus Operator + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `namespaces` | `false` | `[]` | List of namespaces to search for prometheus operator resources in | +| `servicemonitor_namespaces` | `false` | `[]` | List of namespaces to search for just servicemonitors resources in | +| `podmonitor_namespaces` | `false` | `[]` | List of namespaces to search for just podmonitors resources in | +| `probe_namespaces` | `false` | `[]` | List of namespaces to search for just probes resources in | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Push Gateway + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the pushgateway should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=prometheus-pushgateway"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `metrics` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/push-gateway` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./push-gateway.river#L183) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### RabbitMQ + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the rabbitmq should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=rabbitmq-exporter"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `rabbitmq-exporter` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/rabbitmq` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./rabbitmq.river#L183) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Redis + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the redis should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=prometheus-redis-exporter"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `redis-exporter` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/redis` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./redis.river#L183) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Statsd + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the statsd should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=prometheus-statsd-exporter"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `http` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/statsd` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./redis.river#L183) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | + +--- + +### Tempo + +| Argument | Required | Default | Description | +| :------- | :------- | :-------| :---------- | +| `forward_to` | `true` | _NA_ | Must be a list(MetricsReceiver) where collected logs should be forwarded to | +| `enabled` | `false` | `true` | Whether or not the Tempo job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment | +| `namespaces` | `false` | `[]` | The namespaces to look for targets in (`[]` is all namespaces) | +| `selectors` | `false` | `["app.kubernetes.io/name=tempo"]` | The label selectors to use to find matching targets | +| `port_name` | `false` | `http-metrics` | The of the port to scrape metrics from | +| `job_label` | `false` | `integrations/tempo` | The job label to add for all metrics | +| `keep_metrics` | `false` | [see module](./tempo.river#L183) | A regex of metrics to keep | +| `scrape_interval` | `false` | `60s` | How often to scrape metrics from the targets | +| `scrape_timeout` | `false` | `10s` | How long before a scrape times out | +| `max_cache_size` | `false` | `100000` | The maximum number of elements to hold in the relabeling cache. This should be at least 2x-5x your largest scrape target or samples appended rate. | +| `clustering` | `false` | `false` | Whether or not clustering should be enabled | diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/agent.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/agent.river new file mode 100644 index 00000000..f832041e --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/agent.river @@ -0,0 +1,240 @@ +/* +Module: job-agent +Description: Scrapes grafana agent + +Note: Every argument except for "forward_to" is optional, and does have a defined default value. However, the values for these + arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... "). + This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will + does not override the value passed in, where coalesce() will return the first non-null value. +*/ +argument "forward_to" { + comment = "Must be a list(MetricsReceiver) where collected logs should be forwarded to" +} + +argument "enabled" { + comment = "Whether or not the grafana-agent job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment (default: true)" + optional = true +} + +argument "namespaces" { + comment = "The namespaces to look for targets in (default: [] is all namespaces)" + optional = true +} + +argument "selectors" { + // Docs: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + comment = "The label selectors to use to find matching targets (default: [\"app.kubernetes.io/name=grafana-agent\"])" + optional = true +} + +argument "port_name" { + comment = "The of the port to scrape metrics from (default: http-metrics)" + optional = true +} + +argument "job_label" { + comment = "The job label to add for all grafana-agent metric (default: integrations/agent)" + optional = true +} + +argument "keep_metrics" { + comment = "A regex of metrics to keep (default: see below)" + optional = true +} + +argument "scrape_interval" { + comment = "How often to scrape metrics from the targets (default: 60s)" + optional = true +} + +argument "scrape_timeout" { + comment = "How long before a scrape times out (default: 10s)" + optional = true +} + +argument "max_cache_size" { + comment = "The maximum number of elements to hold in the relabeling cache (default: 100000). This should be at least 2x-5x your largest scrape target or samples appended rate." + optional = true +} + +argument "clustering" { + // Docs: https://grafana.com/docs/agent/latest/flow/concepts/clustering/ + comment = "Whether or not clustering should be enabled (default: false)" + optional = true +} + +// grafana agent service discovery for all of the pods in the grafana agent daemonset +discovery.kubernetes "agent" { + role = "pod" + + selectors { + role = "pod" + label = join(coalesce(argument.selectors.value, ["app.kubernetes.io/name=grafana-agent"]), ",") + } + + namespaces { + names = coalesce(argument.namespaces.value, []) + } +} + +// grafana agent relabelings (pre-scrape) +discovery.relabel "agent" { + targets = discovery.kubernetes.agent.targets + + // drop all targets if enabled is false + rule { + target_label = "__enabled" + replacement = format("%s", coalesce(argument.enabled.value, "true")) + } + + rule { + source_labels = ["__enabled"] + regex = "false" + action = "drop" + } + + // keep only the specified metrics port name, and pods that are Running and ready + rule { + source_labels = [ + "__meta_kubernetes_pod_container_port_name", + "__meta_kubernetes_pod_phase", + "__meta_kubernetes_pod_ready", + ] + separator = "@" + regex = coalesce(argument.port_name.value, "http-metrics") + "@Running@true" + action = "keep" + } + + // set the namespace label + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + // set the pod label + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + + // set the container label + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + + // set a workload label + rule { + source_labels = [ + "__meta_kubernetes_pod_controller_kind", + "__meta_kubernetes_pod_controller_name", + ] + separator = "/" + target_label = "workload" + } + + // set the app name if specified as metadata labels "app:" or "app.kubernetes.io/name:" or "k8s-app:" + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_pod_label_app_kubernetes_io_name", + "__meta_kubernetes_pod_label_k8s_app", + "__meta_kubernetes_pod_label_app", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "app" + } +} + +// grafana agent scrape job +prometheus.scrape "agent" { + job_name = coalesce(argument.job_label.value, "integrations/agent") + forward_to = [prometheus.relabel.agent.receiver] + targets = discovery.relabel.agent.output + scrape_interval = coalesce(argument.scrape_interval.value, "60s") + scrape_timeout = coalesce(argument.scrape_timeout.value, "10s") + + clustering { + enabled = coalesce(argument.clustering.value, false) + } +} + +// grafana-agent metric relabelings (post-scrape) +prometheus.relabel "agent" { + forward_to = argument.forward_to.value + max_cache_size = coalesce(argument.max_cache_size.value, 100000) + + // keep only metrics that match the keep_metrics regex + rule { + source_labels = ["__name__"] + regex = coalesce(argument.keep_metrics.value, "(up|log_.+|(agent_(build_info|tcp_connections|wal_(samples_appended_total|storage_active_series))|go_(gc_duration_seconds_count|goroutines|memstats_heap_inuse_bytes)|process_(cpu_seconds_total|start_time_seconds)|prometheus_(remote_storage_(enqueue_retries_total|highest_timestamp_in_seconds|queue_highest_sent_timestamp_seconds|samples_(dropped_total|failed_total|pending|retried_total|total)|sent_batch_duration_seconds_(bucket|count|sum)|shard_(capacity|s(_desired|_max|_min))|succeeded_samples_total)|sd_discovered_targets|target_(interval_length_seconds_(count|sum)|scrapes_(exceeded_sample_limit_total|sample_(duplicate_timestamp_total|out_of_bounds_total|out_of_order_total)))|target_sync_length_seconds_sum|wal_watcher_current_segment)|traces_(exporter_send_failed_spans|exporter_sent_spans|loadbalancer_(backend_outcome|num_backends)|receiver_(accepted_spans|refused_spans))))") + action = "keep" + } + + // remove the component_id label from any metric that starts with log_bytes or log_lines, these are custom metrics that are generated + // as part of the log annotation modules in this repo + rule { + action = "replace" + source_labels = ["__name__"] + regex = "^log_(bytes|lines).+" + replacement = "" + target_label = "component_id" + } + + // set the namespace label to that of the exported_namespace + rule { + action = "replace" + source_labels = ["__name__", "exported_namespace"] + separator = "@" + regex = "^log_(bytes|lines).+@(.+)" + replacement = "$2" + target_label = "namespace" + } + + // set the pod label to that of the exported_pod + rule { + action = "replace" + source_labels = ["__name__", "exported_pod"] + separator = "@" + regex = "^log_(bytes|lines).+@(.+)" + replacement = "$2" + target_label = "pod" + } + + // set the container label to that of the exported_container + rule { + action = "replace" + source_labels = ["__name__", "exported_container"] + separator = "@" + regex = "^log_(bytes|lines).+@(.+)" + replacement = "$2" + target_label = "container" + } + + // set the job label to that of the exported_job + rule { + action = "replace" + source_labels = ["__name__", "exported_job"] + separator = "@" + regex = "^log_(bytes|lines).+@(.+)" + replacement = "$2" + target_label = "job" + } + + // set the instance label to that of the exported_instance + rule { + action = "replace" + source_labels = ["__name__", "exported_instance"] + separator = "@" + regex = "^log_(bytes|lines).+@(.+)" + replacement = "$2" + target_label = "instance" + } + + rule { + action = "labeldrop" + regex = "exported_(namespace|pod|container|job|instance)" + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/annotations-probe.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/annotations-probe.river new file mode 100644 index 00000000..694c4565 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/annotations-probe.river @@ -0,0 +1,466 @@ +/* +Module: job-agent +Description: Scrapes grafana agent + +Note: Every argument except for "forward_to" and "role" is optional, and does have a defined default value. However, the values for these + arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... "). + This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will + does not override the value passed in, where coalesce() will return the first non-null value. + +Kubernetes Service Auto-Probing +------------------------------------------------------------------------------------------------------------------------------------ +This module is meant to be used to automatically scrape targets based on a certain role and set of annotations. This module can be consumed +multiple times with different roles. The supported roles are: + + - service + - ingress + +Each port attached to an service is an eligible target, oftentimes service will have multiple ports. +There may be instances when you want to probe all ports or some ports and not others. To support this +the following annotations are available: + +only probe services with probe set to true, this can be single valued i.e. probe all ports for +the service: + +probes.grafana.com/probe: true + +the default probing scheme is "", this can be specified as a single value which would override, +if using HTTP prober specify "http" or "https": + +probes.grafana.com/scheme: https + +the default path to probe is /metrics, this can be specified as a single value which would override, +the probe path being used for all ports attached to the service: + +probes.grafana.com/path: /metrics/some_path + +the default module to use for probing the default value is "unknown" as the modules are defined are in your blackbox exporter +configuration file, this can be specified as a single value which would override, the probe module being used for all ports +attached to the service: + +probes.grafana.com/module: http_2xx + +the default port to probe is the service port, this can be specified as a single value which would +override the probe port being used for all ports attached to the service, note that even if aan service had +multiple targets, the relabel_config targets are deduped before scraping: + +probes.grafana.com/port: 8080 + +the value to set for the job label, by default this would be "integrations/blackbox_exporter" if not specified: + +probes.grafana.com/job: blackbox-exporter + +the default interval to probe is 1m, this can be specified as a single value which would override, +the probe interval being used for all ports attached to the service: + +probes.grafana.com/interval: 5m + +the default timeout for scraping is 10s, this can be specified as a single value which would override, +the probe interval being used for all ports attached to the service: + +probes.grafana.com/timeout: 30s +*/ +argument "forward_to" { + comment = "Must be a list(MetricsReceiver) where collected logs should be forwarded to" +} + +argument "role" { + comment = "The role to use when looking for targets to scrape via annotations, can be: service or ingress (default: service)" +} + +argument "blackbox_url" { + comment = "The address of the blackbox exporter to use (without the protocol), only the hostname and port i.e. blackbox-prometheus-blackbox-exporter.default.svc.cluster.local:9115" +} + +argument "namespaces" { + comment = "The namespaces to look for targets in (default: [] is all namespaces)" + optional = true +} + +argument "selectors" { + // see: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + comment = "The label selectors to use to find matching annotation targets (default: [] is all targets)" + optional = true +} + +argument "annotation" { + // Docs: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + // k8s selectors d not support a logical OR, if multiple types of annotations are needed, this module should be invoked multiple times + // i.e. probes.grafana.com, then again for prometheus.io + comment = "The annotation namespace to use (default: probes.grafana.com)" + default = "probes.grafana.com" + optional = true +} + +argument "tenant" { + comment = "The tenant to write metrics to. This does not have to be the tenantId, this is the value to look for in the probes.grafana.com/tenant annotation, and this can be a regex." + optional = true + default = ".*" +} + +argument "keep_metrics" { + comment = "A regex of metrics to keep (default: (.+))" + optional = true +} + +argument "drop_metrics" { + comment = "A regex of metrics to drop (default: \"\")" + optional = true +} + +argument "scrape_interval" { + comment = "How often to scrape metrics from the targets (default: 60s)" + optional = true +} + +argument "scrape_timeout" { + comment = "How long before a scrape times out (default: 10s)" + optional = true +} + +argument "max_cache_size" { + comment = "The maximum number of elements to hold in the relabeling cache (default: 100000). This should be at least 2x-5x your largest scrape target or samples appended rate." + optional = true +} + +argument "clustering" { + // Docs: https://grafana.com/docs/agent/latest/flow/concepts/clustering/ + comment = "Whether or not clustering should be enabled (default: false)" + optional = true +} + +/* + Hidden Arguments + These arguments are used to set reusable variables to avoid repeating logic +*/ +argument "__sd_annotation" { + optional = true + comment = "The logic is used to transform the annotation argument into a valid label name by removing unsupported characters." + default = replace(replace(replace(coalesce(argument.annotation.value, "probes.grafana.com"), ".", "_"), "/", "_"), "-", "_") +} + +// annotations service discovery +discovery.kubernetes "probes" { + role = coalesce(argument.role.value, "service") + + selectors { + role = coalesce(argument.role.value, "service") + label = join(coalesce(argument.selectors.value, []), ",") + } + + namespaces { + names = coalesce(argument.namespaces.value, []) + } +} + +discovery.relabel "probes" { + targets = discovery.kubernetes.probes.targets + + /**************************************************************************************************************** + * Handle Targets to Keep or Drop + ****************************************************************************************************************/ + // allow resources to declare they should be probed or not + // Example Annotation: + // probes.grafana.com/probe: false + // + // the label prometheus.io/service-monitor: "false" is a common label for headless services, if it is set to false, + // do not probe the target + rule { + action = "keep" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_probe", + "__meta_kubernetes_" + argument.role.value + "_label_prometheus_io_service_monitor", + ] + regex = "^true;(|true)$" + } + + // only keep targets where the pod is running or the pod_phase is empty and is not an init container. This will only exist for role="pod" or + // potentially role="endpoints", if it is a service the value is empty and thus allowed to pass, if it is an endpoint but not associated to a + // pod but rather a static IP or hostname, that could be outside of kubernetes allow endpoints to declare what tenant their metrics should be + // written to + rule { + action = "keep" + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "^(?i)(Running|)$" + } + + rule { + action = "keep" + source_labels = ["__meta_kubernetes_pod_ready"] + regex = "^(true|)$" + } + // if the container is an init container, drop it + rule { + action = "drop" + source_labels = ["__meta_kubernetes_pod_container_init"] + regex = "^(true)$" + } + + // allow resources to declare their metrics the tenant their metrics should be sent to, + // Example Annotation: + // probes.grafana.com/tenant: primary + // + // Note: This does not necessarily have to be the actual tenantId, it can be a friendly name as well that is simply used + // to determine if the metrics should be gathered for the current tenant + rule { + action = "keep" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_tenant", + ] + regex = "^(" + argument.tenant.value + ")$" + } + + /**************************************************************************************************************** + * Handle Setting Scrape Metadata i.e. path, port, interval etc. + ****************************************************************************************************************/ + // allow resources to declare the protocol to use when collecting metrics, the default value is "http", this is the scheme + // of the target address, not the scheme to use for blackbox exporter + // Example Annotation: + // probes.grafana.com/scheme: http + rule { + action = "replace" + replacement = "http" + target_label = "__scheme__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_scheme", + "__meta_kubernetes_ingress_scheme", // this would only exist for ingresses and can be used instead of setting the annotation + ] + separator = ";" + regex = "^(?:;*)?(https?).*$" + replacement = "$1" + target_label = "__scheme__" + } + + // allow resources to declare the port to use when collecting metrics, the default value is the discovered port from + // Example Annotation: + // probes.grafana.com/port: 9090 + rule { + action = "replace" + source_labels = [ + "__address__", + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_port", + "__meta_kubernetes_" + argument.role.value + "_port_number", + ] + separator = ";" + regex = "^([^:]+)(?::\\d+)?(?:;*)?([^;]+).*" + replacement = "$1:$2" + target_label = "__address__" + } + + // allow resources to declare their the path to use when collecting their metrics, the default value is "/metrics", + // Example Annotation: + // probes.grafana.com/path: /metrics/foo + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_path", + "__meta_kubernetes_ingress_path", // this would only exist for ingresses and can be used instead of setting the annotation + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__metrics_path__" + } + + // set the target address to probe + rule { + action = "replace" + source_labels = [ + "__scheme__", + "__address__", + "__metrics_path__", + ] + separator = ";" + regex = "(.*);(.+);(.+)" + replacement = "${1}://${2}${3}" + target_label = "__param_target" + } + + // allow resources to declare their the module to use when probing, the default value is "unknown", + // Example Annotation: + // probes.grafana.com/module: http_2xx + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_module", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__param_module" + } + + // allow resources to declare how often their metrics should be collected, the default value is 1m, + // the following duration formats are supported (s|m|ms|h|d): + // Example Annotation: + // probes.grafana.com/interval: 5m + rule { + action = "replace" + replacement = coalesce(argument.scrape_interval.value, "60s") + target_label = "__scrape_interval__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_interval", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_interval__" + } + + // allow resources to declare the timeout of the scrape request, the default value is 10s, + // the following duration formats are supported (s|m|ms|h|d): + // Example Annotation: + // probes.grafana.com/timeout: 30s + rule { + action = "replace" + replacement = coalesce(argument.scrape_timeout.value, "10s") + target_label = "__scrape_timeout__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_timeout", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_timeout__" + } + + /**************************************************************************************************************** + * Handle Setting Common Labels + ****************************************************************************************************************/ + // set the instance label to the target + rule { + action = "replace" + source_labels = ["__param_target"] + target_label = "instance" + } + + // ensure the __metrics_path is set to /probe + rule { + action = "replace" + replacement = "/probe" + target_label = "__metrics_path__" + } + + // set the __address__ to send the scrape request to be the probing exporter service address that has been deployed + rule { + action = "replace" + replacement = argument.blackbox_url.value + target_label = "__address__" + } + + // set the namespace label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + // set the target name label i.e. service name, ingress name, etc. + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_name", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = argument.role.value + } + + // set a default job label to be the namespace/service_name or namespace/ingress_name + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_namespace", + argument.role.value, + ] + separator = ";" + regex = "^([^;]+)(?:;*)?([^;]+).*$" + replacement = "$1/$2" + target_label = "job" + } + + // allow resources to declare their the job label value to use when collecting their metrics, the default value is "", + // Example Annotation: + // probes.grafana.com/job: my-service/ready-probe + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_job", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "job" + } + + // set the app name if specified as metadata labels "app:" or "app.kubernetes.io/name:" + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_label_app_kubernetes_io_name", + "__meta_kubernetes_" + argument.role.value + "_label_k8s_app", + "__meta_kubernetes_" + argument.role.value + "_label_app", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "app" + } + + // set the app component if specified as metadata labels "component:" or "app.kubernetes.io/component:" + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_label_app_kubernetes_io_component", + "__meta_kubernetes_" + argument.role.value + "_label_component", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "component" + } +} + +// scrape http only targtets +prometheus.scrape "probes" { + job_name = "annotation-probe-http" + forward_to = [prometheus.relabel.probes.receiver] + targets = discovery.relabel.probes.output + scheme = "http" + scrape_interval = coalesce(argument.scrape_interval.value, "60s") + scrape_timeout = coalesce(argument.scrape_timeout.value, "10s") + + clustering { + enabled = coalesce(argument.clustering.value, false) + } +} + +// perform generic relabeling using keep_metrics and drop_metrics +prometheus.relabel "probes" { + forward_to = argument.forward_to.value + + // keep only metrics that match the keep_metrics regex + rule { + action = "keep" + source_labels = ["__name__"] + regex = coalesce(argument.keep_metrics.value, "(.+)") + } + + // drop metrics that match the drop_metrics regex + rule { + action = "drop" + source_labels = ["__name__"] + regex = coalesce(argument.drop_metrics.value, "") + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/annotations-scrape.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/annotations-scrape.river new file mode 100644 index 00000000..f9b2b87f --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/annotations-scrape.river @@ -0,0 +1,574 @@ +/* +Module: job-annotation-scrape +Description: Scrapes targets for metrics based on annotations + +Note: Every argument except for "forward_to" and "role" is optional, and does have a defined default value. However, the values for these + arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... "). + This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will + does not override the value passed in, where coalesce() will return the first non-null value. + +Kubernetes Annotation Auto-Scraping +------------------------------------------------------------------------------------------------------------------------------------ +This module is meant to be used to automatically scrape targets based on a certain role and set of annotations. This module can be consumed +multiple times with different roles. The supported roles are: + + - pod + - service + - endpoints + +Typically, if mimicking the behavior of the prometheus-operator, and ServiceMonitor functionality you would use role="endpoints", if +mimicking the behavior of the PodMonitor functionality you would use role="pod". It is important to understand that with endpoints, +the target is typically going to be a pod, and whatever annotations that are set on the service will automatically be propagated to the +endpoints. This is why the role "endpoints" is used, because it will scrape the pod, but also consider the service annotations. Using +role="endpoints", which scrape each endpoint associated to the service. If role="service" is used, it will only scrape the service, only +hitting one of the endpoints associated to the service. + +This is where you must consider your scraping strategy, for example if you scrape a service like "kube-state-metrics" using +role="endpoints" you should only have a single replica of the kube-state-metrics pod, if you have multiple replicas, you should use +role="service" or a separate non-annotation job completely. Scraping a service instead of endpoints, is typically a rare use case, but +it is supported. + +There are other considerations for using annotation based scraping as well, which is metric relabeling rules that happen post scrape. If +you have a target that you want to apply a bunch of relabelings to or a very large metrics response payload, performance wise it will be +better to have a separate job for that target, rather than using use annotations. As every targert will go through the ssame relabeling. +Typical deployment strategies/options would be: + +Option #1 (recommended): + - Annotation Scraping for role="endpoints" + - Separate Jobs for specific service scrapes (i.e. kube-state-metrics, node-exporter, etc.) or large metric payloads + - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, kube-scheduler, etc.) + +Option #2: + - Annotation Scraping for role="pod" + - Annotation Scraping for role="service" (i.e. kube-state-metrics, node-exporter, etc.) + - Separate Jobs for specific use cases or large metric payloads + - Separate Jobs for K8s API scraping (i.e. cadvisor, kube-apiserver, kube-scheduler, etc.) + +At no point should you use role="endpoints" and role="pod" together, as this will result in duplicate targets being scraped, thus +generating duplicate metrics. If you want to scrape both the pod and the service, use Option #2. + +Each port attached to an service/pod/endpoint is an eligible target, oftentimes it will have multiple ports. +There may be instances when you want to scrape all ports or some ports and not others. To support this +the following annotations are available: + + metrics.agent.grafana.com/scrape: true + +the default scraping scheme is http, this can be specified as a single value which would override, the schema being used for all +ports attached to the target: + + metrics.agent.grafana.com/scheme: https + +the default path to scrape is /metrics, this can be specified as a single value which would override, the scrape path being used +for all ports attached to the target: + + metrics.agent.grafana.com/path: /metrics/some_path + +the default port to scrape is the target port, this can be specified as a single value which would override the scrape port being +used for all ports attached to the target, note that even if aan target had multiple targets, the relabel_config targets are +deduped before scraping: + + metrics.agent.grafana.com/port: 8080 + +the default interval to scrape is 1m, this can be specified as a single value which would override, the scrape interval being used +for all ports attached to the target: + + metrics.agent.grafana.com/interval: 5m + +the default timeout for scraping is 10s, this can be specified as a single value which would override, the scrape interval being +used for all ports attached to the target: + + metrics.agent.grafana.com/timeout: 30s + +the default job is namespace/{{ service name }} or namespace/{{ controller_name }} depending on the role, there may be instantces +in which a different job name is required because of a set of dashboards, rules, etc. to support this there is a job annotation +which will override the default value: + + metrics.agent.grafana.com/job: integrations/kubernetes/kube-state-metrics +*/ +argument "forward_to" { + comment = "Must be a list(MetricsReceiver) where collected logs should be forwarded to" +} + +argument "role" { + comment = "The role to use when looking for targets to scrape via annotations, can be: endpoints, service, pod (default: endpoints)" +} + +argument "namespaces" { + comment = "The namespaces to look for targets in (default: [] is all namespaces)" + optional = true +} + +argument "selectors" { + // see: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + comment = "The label selectors to use to find matching annotation targets (default: [] is all targets)" + optional = true +} + +argument "annotation" { + // Docs: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + // k8s selectors d not support a logical OR, if multiple types of annotations are needed, this module should be invoked multiple times + // i.e. metrics.agent.grafana.com, then again for prometheus.io + comment = "The annotation namespace to use (default: metrics.agent.grafana.com)" + default = "metrics.agent.grafana.com" + optional = true +} + +argument "tenant" { + comment = "The tenant to write metrics to. This does not have to be the tenantId, this is the value to look for in the logs.agent.grafana.com/tenant annotation, and this can be a regex." + optional = true + default = ".*" +} + +argument "keep_metrics" { + comment = "A regex of metrics to keep (default: (.+))" + optional = true +} + +argument "drop_metrics" { + comment = "A regex of metrics to drop (default: \"\")" + optional = true +} + +argument "scrape_port_named_metrics" { + comment = "Whether or not to automatically scrape endpoints that have a port with 'metrics' in the name" + optional = true + default = false +} + +argument "scrape_interval" { + comment = "How often to scrape metrics from the targets (default: 60s)" + optional = true +} + +argument "scrape_timeout" { + comment = "How long before a scrape times out (default: 10s)" + optional = true +} + +argument "max_cache_size" { + comment = "The maximum number of elements to hold in the relabeling cache (default: 100000). This should be at least 2x-5x your largest scrape target or samples appended rate." + optional = true +} + +argument "clustering" { + // Docs: https://grafana.com/docs/agent/latest/flow/concepts/clustering/ + comment = "Whether or not clustering should be enabled (default: false)" + optional = true +} + +/* + Hidden Arguments + These arguments are used to set reusable variables to avoid repeating logic +*/ +argument "__pod_role" { + comment = "Most annotation targets service or pod that is all you want, however if the role is endpoints you want the pod" + optional = true + default = replace(coalesce(argument.role.value, "endpoints"), "endpoints", "pod") +} + +argument "__service_role" { + comment = "Most annotation targets service or pod that is all you want, however if the role is endpoints you we also want to consider service annotations" + optional = true + default = replace(coalesce(argument.role.value, "endpoints"), "endpoints", "service") +} + +argument "__sd_annotation" { + optional = true + comment = "The logic is used to transform the annotation argument into a valid label name by removing unsupported characters." + default = replace(replace(replace(coalesce(argument.annotation.value, "metrics.agent.grafana.com"), ".", "_"), "/", "_"), "-", "_") +} + +// annotations service discovery +discovery.kubernetes "annotations" { + role = coalesce(argument.role.value, "endpoints") + + selectors { + role = coalesce(argument.role.value, "endpoints") + label = join(coalesce(argument.selectors.value, []), ",") + } + + namespaces { + names = coalesce(argument.namespaces.value, []) + } +} + +discovery.relabel "annotations" { + targets = discovery.kubernetes.annotations.targets + + /**************************************************************************************************************** + * Handle Targets to Keep or Drop + ****************************************************************************************************************/ + // allow resources to declare their metrics scraped or not + // Example Annotation: + // metrics.agent.grafana.com/scrape: false + // + // the label prometheus.io/service-monitor: "false" is a common label for headless services, when performing endpoint + // service discovery, if there is both a load-balanced service and headless service, this can result in duplicate + // scrapes if the name of the service is attached as a label. any targets with this label or annotation set should be dropped + rule { + action = "replace" + replacement = "false" + target_label = "__tmp_scrape" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_scrape", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_scrape", + "__meta_kubernetes_" + argument.role.value + "_label_prometheus_io_service_monitor", + ] + separator = ";" + // only allow empty or true, otherwise defaults to false + regex = "^(?:;*)?(true)(;|true)*$" + replacement = "$1" + target_label = "__tmp_scrape" + } + + // add a __tmp_scrape_port_named_metrics from the argument.scrape_port_named_metrics + rule { + replacement = format("%t", argument.scrape_port_named_metrics.value) + target_label = "__tmp_scrape_port_named_metrics" + } + + // only keep targets that have scrape: true or "metrics" in the port name if the argument scrape_port_named_metrics + rule { + action = "keep" + source_labels = [ + "__tmp_scrape", + "__tmp_scrape_port_named_metrics", + // endpoints is the role and most meta labels started with "endpoints", however the port name is an exception and starts with "endpoint" + "__meta_kubernetes_" + replace(coalesce(argument.role.value, "endpoints"), "endpoints", "endpoint") + "_port_name", + ] + separator = ";" + regex = "^(true;.*|(|true);true;(.*metrics.*))$" + } + + // only keep targets where the pod is running or the pod_phase is empty and is not an init container. This will only exist for role="pod" or + // potentially role="endpoints", if it is a service the value is empty and thus allowed to pass, if it is an endpoint but not associated to a + // pod but rather a static IP or hostname, that could be outside of kubernetes allow endpoints to declare what tenant their metrics should be + // written to + rule { + action = "keep" + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "^(?i)(Running|)$" + } + + rule { + action = "keep" + source_labels = ["__meta_kubernetes_pod_ready"] + regex = "^(true|)$" + } + // if the container is an init container, drop it + rule { + action = "drop" + source_labels = ["__meta_kubernetes_pod_container_init"] + regex = "^(true)$" + } + + // allow resources to declare their metrics the tenant their metrics should be sent to, + // Example Annotation: + // metrics.agent.grafana.com/tenant: primary + // + // Note: This does not necessarily have to be the actual tenantId, it can be a friendly name as well that is simply used + // to determine if the metrics should be gathered for the current tenant + rule { + action = "keep" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_tenant", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_tenant", + ] + regex = "^(" + argument.tenant.value + ")$" + } + + /**************************************************************************************************************** + * Handle Setting Scrape Metadata i.e. path, port, interval etc. + ****************************************************************************************************************/ + // allow resources to declare the protocol to use when collecting metrics, the default value is "http", + // Example Annotation: + // metrics.agent.grafana.com/scheme: http + rule { + action = "replace" + replacement = "http" + target_label = "__scheme__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_scheme", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_scheme", + ] + separator = ";" + regex = "^(?:;*)?(https?).*$" + replacement = "$1" + target_label = "__scheme__" + } + + // allow resources to declare the port to use when collecting metrics, the default value is the discovered port from + // Example Annotation: + // metrics.agent.grafana.com/port: 9090 + rule { + action = "replace" + source_labels = [ + "__address__", + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_port", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_port", + ] + separator = ";" + regex = "^([^:]+)(?::\\d+)?;(\\d+)$" + replacement = "$1:$2" + target_label = "__address__" + } + + // allow resources to declare their the path to use when collecting their metrics, the default value is "/metrics", + // Example Annotation: + // metrics.agent.grafana.com/path: /metrics/foo + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_path", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_path", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "__metrics_path__" + } + + // allow resources to declare how often their metrics should be collected, the default value is 1m, + // the following duration formats are supported (s|m|ms|h|d): + // Example Annotation: + // metrics.agent.grafana.com/interval: 5m + rule { + action = "replace" + replacement = coalesce(argument.scrape_interval.value, "60s") + target_label = "__scrape_interval__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_interval", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_interval", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_interval__" + } + + // allow resources to declare the timeout of the scrape request, the default value is 10s, + // the following duration formats are supported (s|m|ms|h|d): + // Example Annotation: + // metrics.agent.grafana.com/timeout: 30s + rule { + action = "replace" + replacement = coalesce(argument.scrape_timeout.value, "10s") + target_label = "__scrape_timeout__" + } + + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_timeout", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_timeout", + ] + separator = ";" + regex = "^(?:;*)?(\\d+(s|m|ms|h|d)).*$" + replacement = "$1" + target_label = "__scrape_timeout__" + } + + /**************************************************************************************************************** + * Handle Setting Common Labels + ****************************************************************************************************************/ + // set the namespace label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + // set the target name label i.e. service name, pod name, etc. + // if the role is endpoints, the first valued field is used which would be __meta_kubernetes_pod_name, if the pod name is empty + // then the endpoint name would be used + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.__pod_role.value + "_name", + "__meta_kubernetes_" + argument.role.value + "_name", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = argument.__pod_role.value + } + + // set a default job label to be the namespace/pod_controller_name or namespace/service_name + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_namespace", + "__meta_kubernetes_pod_controller_name", + argument.__pod_role.value, + ] + separator = ";" + regex = "^([^;]+)(?:;*)?([^;]+).*$" + replacement = "$1/$2" + target_label = "job" + } + + // if the controller is a ReplicaSet, drop the hash from the end of the ReplicaSet + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_pod_controller_type", + "__meta_kubernetes_namespace", + "__meta_kubernetes_pod_controller_name", + ] + separator = ";" + regex = "^(?:ReplicaSet);([^;]+);([^;]+)-.+$" + replacement = "$1/$2" + target_label = "job" + } + + // allow resources to declare their the job label value to use when collecting their metrics, the default value is "", + // Example Annotation: + // metrics.agent.grafana.com/job: integrations/kubernetes/cadvisor + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_annotation_" + argument.__sd_annotation.value + "_job", + "__meta_kubernetes_" + argument.__service_role.value + "_annotation_" + argument.__sd_annotation.value + "_job", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "job" + } + + // set the app name if specified as metadata labels "app:" or "app.kubernetes.io/name:" + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_label_app_kubernetes_io_name", + "__meta_kubernetes_" + argument.role.value + "_label_k8s_app", + "__meta_kubernetes_" + argument.role.value + "_label_app", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_app_kubernetes_io_name", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_k8s_app", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_app", + "__meta_kubernetes_" + argument.__service_role.value + "_label_app_kubernetes_io_name", + "__meta_kubernetes_" + argument.__service_role.value + "_label_k8s_app", + "__meta_kubernetes_" + argument.__service_role.value + "_label_app", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "app" + } + + // set the app component if specified as metadata labels "component:" or "app.kubernetes.io/component:" + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_" + argument.role.value + "_label_app_kubernetes_io_component", + "__meta_kubernetes_" + argument.role.value + "_label_component", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_app_kubernetes_io_component", + "__meta_kubernetes_" + argument.__pod_role.value + "_label_component", + "__meta_kubernetes_" + argument.__service_role.value + "_label_app_kubernetes_io_component", + "__meta_kubernetes_" + argument.__service_role.value + "_label_component", + ] + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "component" + } + + // add a workload label if the resource is a pod + // example: grafana-agent-68nv9 becomes DaemonSet/grafana-agent + rule { + source_labels = [ + "__meta_kubernetes_pod_controller_kind", + "__meta_kubernetes_pod_controller_name", + ] + action = "replace" + regex = "^(.+);(.+)$" + replacement = "$1/$2" + target_label = "controller" + } +} + +// only keep http targets +discovery.relabel "http_annotations" { + targets = discovery.relabel.annotations.output + + rule { + action = "keep" + source_labels = ["__scheme__"] + regex = "http" + } +} + +// scrape http only targtets +prometheus.scrape "http_annotations" { + job_name = "annotation-metrics-http" + forward_to = [prometheus.relabel.annotations.receiver] + targets = discovery.relabel.http_annotations.output + scheme = "http" + scrape_interval = coalesce(argument.scrape_interval.value, "60s") + scrape_timeout = coalesce(argument.scrape_timeout.value, "10s") + + clustering { + enabled = coalesce(argument.clustering.value, false) + } +} + +// only keep https targets +discovery.relabel "https_annotations" { + targets = discovery.relabel.annotations.output + + rule { + action = "keep" + source_labels = ["__scheme__"] + regex = "https" + } +} + +// scrape https only targtets +prometheus.scrape "https_annotations" { + job_name = "annotation-metrics-https" + forward_to = [prometheus.relabel.annotations.receiver] + targets = discovery.relabel.https_annotations.output + scheme = "https" + scrape_interval = coalesce(argument.scrape_interval.value, "60s") + scrape_timeout = coalesce(argument.scrape_timeout.value, "10s") + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + tls_config { + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + insecure_skip_verify = false + server_name = "kubernetes" + } + + clustering { + enabled = coalesce(argument.clustering.value, false) + } +} + +// perform generic relabeling using keep_metrics and drop_metrics +prometheus.relabel "annotations" { + forward_to = argument.forward_to.value + + // keep only metrics that match the keep_metrics regex + rule { + action = "keep" + source_labels = ["__name__"] + regex = coalesce(argument.keep_metrics.value, "(.+)") + } + + // drop metrics that match the drop_metrics regex + rule { + action = "drop" + source_labels = ["__name__"] + regex = coalesce(argument.drop_metrics.value, "") + } +} diff --git a/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/cadvisor.river b/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/cadvisor.river new file mode 100644 index 00000000..b159dc20 --- /dev/null +++ b/kubernetes/common/grafana-agent/configs/modules/kubernetes/metrics/jobs/cadvisor.river @@ -0,0 +1,196 @@ +/* +Module: job-cadvisor +Description: Scrapes cAdvisor (Container Advisor Metrics) + +Note: Every argument except for "forward_to" is optional, and does have a defined default value. However, the values for these + arguments are not defined using the default = " ... " argument syntax, but rather using the coalesce(argument.value, " ... "). + This is because if the argument passed in from another consuming module is set to null, the default = " ... " syntax will + does not override the value passed in, where coalesce() will return the first non-null value. +*/ +argument "forward_to" { + comment = "Must be a list(MetricsReceiver) where collected logs should be forwarded to" +} + +argument "enabled" { + comment = "Whether or not the cadvisor job should be enabled, this is useful for disabling the job when it is being consumed by other modules in a multi-tenancy environment (default: true)" + optional = true +} + +argument "job_label" { + comment = "The job label to add for all cadvisor metrics (default: integrations/kubernetes/cadvisor)" + optional = true +} + +argument "keep_metrics" { + comment = "A regex of metrics to keep (default: see below)" + optional = true +} + +argument "scrape_interval" { + comment = "How often to scrape metrics from the targets (default: 60s)" + optional = true +} + +argument "scrape_timeout" { + comment = "How long before a scrape times out (default: 10s)" + optional = true +} + +argument "max_cache_size" { + comment = "The maximum number of elements to hold in the relabeling cache (default: 100000). This should be at least 2x-5x your largest scrape target or samples appended rate." + optional = true +} + +argument "clustering" { + // Docs: https://grafana.com/docs/agent/latest/flow/concepts/clustering/ + comment = "Whether or not clustering should be enabled (default: false)" + optional = true +} + +// cadvisor service discovery +discovery.kubernetes "cadvisor" { + role = "node" +} + +// cadvisor relabelings (pre-scrape) +discovery.relabel "cadvisor" { + targets = discovery.kubernetes.cadvisor.targets + + // drop all targets if enabled is false + rule { + target_label = "__enabled" + replacement = format("%s", coalesce(argument.enabled.value, "true")) + } + + rule { + source_labels = ["__enabled"] + regex = "false" + action = "drop" + } + + // set the address to use the kubernetes service dns name + rule { + target_label = "__address__" + replacement = "kubernetes.default.svc.cluster.local:443" + } + + // set the metrics path to use the proxy path to the nodes cadvisor metrics endpoint + rule { + source_labels = ["__meta_kubernetes_node_name"] + regex = "(.+)" + replacement = "/api/v1/nodes/${1}/proxy/metrics/cadvisor" + target_label = "__metrics_path__" + } +} + +// cadvisor scrape job +prometheus.scrape "cadvisor" { + job_name = coalesce(argument.job_label.value, "integrations/kubernetes/cadvisor") + forward_to = [prometheus.relabel.cadvisor.receiver] + targets = discovery.relabel.cadvisor.output + scheme = "https" + scrape_interval = coalesce(argument.scrape_interval.value, "60s") + scrape_timeout = coalesce(argument.scrape_timeout.value, "10s") + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + tls_config { + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + insecure_skip_verify = false + server_name = "kubernetes" + } + + clustering { + enabled = coalesce(argument.clustering.value, false) + } +} + +// cadvisor metric relabelings (post-scrape) +prometheus.relabel "cadvisor" { + forward_to = argument.forward_to.value + max_cache_size = coalesce(argument.max_cache_size.value, 100000) + + // keep only metrics that match the keep_metrics regex + rule { + source_labels = ["__name__"] + regex = coalesce(argument.keep_metrics.value, "(up|container_(cpu_(cfs_(periods|throttled_periods)_total|usage_seconds_total)|fs_(reads|writes)(_bytes)?_total|memory_(cache|rss|swap|working_set_bytes)|network_(receive|transmit)_(bytes|packets(_dropped)?_total))|machine_memory_bytes)") + action = "keep" + } + + // Drop empty container labels, addressing https://github.com/google/cadvisor/issues/2688 + rule { + source_labels = ["__name__", "container"] + separator = "@" + regex = "(container_cpu_.*|container_fs_.*|container_memory_.*)@" + action = "drop" + } + + // Drop empty image labels, addressing https://github.com/google/cadvisor/issues/2688 + rule { + source_labels = ["__name__", "image"] + separator = "@" + regex = "(container_cpu_.*|container_fs_.*|container_memory_.*|container_network_.*)@" + action = "drop" + } + + // Normalizing unimportant labels (not deleting to continue satisfying