From f4679bd7fbd114160dcce73e21fa674cfde38c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Thu, 21 Sep 2023 17:32:45 +0200 Subject: [PATCH] chore(metrics): route based on job attributes We can do this after having consolidated our remote writes. --- .../conf/metrics/otelcol/config.yaml | 1 - .../conf/metrics/otelcol/pipeline.yaml | 3 - .../conf/metrics/otelcol/processors.yaml | 56 +++---------------- tests/helm/metrics_test.go | 3 - .../additional_endpoints.output.yaml | 49 +++------------- .../metadata_metrics_otc/basic.output.yaml | 49 +++------------- .../metadata_metrics_otc/custom.output.yaml | 8 --- .../filtered_app_metrics.output.yaml | 49 +++------------- 8 files changed, 36 insertions(+), 182 deletions(-) diff --git a/deploy/helm/sumologic/conf/metrics/otelcol/config.yaml b/deploy/helm/sumologic/conf/metrics/otelcol/config.yaml index 9372a63eba..fdce4035c3 100644 --- a/deploy/helm/sumologic/conf/metrics/otelcol/config.yaml +++ b/deploy/helm/sumologic/conf/metrics/otelcol/config.yaml @@ -34,7 +34,6 @@ receivers: write_timeout = "30s" service_address = ":9888" data_format = "prometheusremotewrite" - path_tag = true paths = [ {{ include "metric.endpoints" . | indent 10 }} ] diff --git a/deploy/helm/sumologic/conf/metrics/otelcol/pipeline.yaml b/deploy/helm/sumologic/conf/metrics/otelcol/pipeline.yaml index 65cf8b346b..1df20b2ebe 100644 --- a/deploy/helm/sumologic/conf/metrics/otelcol/pipeline.yaml +++ b/deploy/helm/sumologic/conf/metrics/otelcol/pipeline.yaml @@ -33,10 +33,7 @@ processors: {{- end }} - batch {{- if eq .Values.sumologic.metrics.sourceType "http" }} - - transform/prepare_routing - routing -{{- else }} - - transform/drop_routing_attribute {{- end }} receivers: - telegraf diff --git a/deploy/helm/sumologic/conf/metrics/otelcol/processors.yaml b/deploy/helm/sumologic/conf/metrics/otelcol/processors.yaml index ca12ae4c17..8d39f501a0 100644 --- a/deploy/helm/sumologic/conf/metrics/otelcol/processors.yaml +++ b/deploy/helm/sumologic/conf/metrics/otelcol/processors.yaml @@ -132,57 +132,42 @@ resource/remove_k8s_pod_pod_name: key: k8s.pod.pod_name {{- if eq .Values.sumologic.metrics.sourceType "http" }} -## NOTE: below listed rules could be simplified if routingprocessor -## supports regex matching. At this point we could group route entries -## going to the same set of exporters. routing: - attribute_source: resource default_exporters: - sumologic/default - drop_resource_routing_attribute: true - from_attribute: http_listener_v2_path + error_mode: ignore table: ## apiserver metrics - exporters: - sumologic/apiserver - value: /prometheus.metrics.apiserver + statement: route() where resource.attributes["job"] == "apiserver" ## control-plane metrics - exporters: - sumologic/control_plane - value: /prometheus.metrics.control-plane.coredns + statement: route() where resource.attributes["job"] == "coredns" - exporters: - sumologic/control_plane - value: /prometheus.metrics.control-plane.kube-etcd + statement: route() where resource.attributes["job"] == "kube-etcd" ## controller metrics - exporters: - sumologic/controller - value: /prometheus.metrics.controller-manager + statement: route() where resource.attributes["job"] == "kube-controller-manager" ## kubelet metrics - exporters: - sumologic/kubelet - value: /prometheus.metrics.kubelet + statement: route() where resource.attributes["job"] == "kubelet" ## node metrics - exporters: - sumologic/node - value: /prometheus.metrics.node + statement: route() where resource.attributes["job"] == "node-exporter" ## scheduler metrics - exporters: - sumologic/scheduler - value: /prometheus.metrics.scheduler + statement: route() where resource.attributes["job"] == "kube-scheduler" ## state metrics - exporters: - sumologic/state - value: /prometheus.metrics.state - ## custom metrics - ## These entries are necessary due to a bug in routing processor that prevents the routing key from being deleted - ## if the default exporter is chosen - ## See: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/24644 - - exporters: - - sumologic/default - value: /prometheus.metrics.applications.custom - - exporters: - - sumologic/default - value: /prometheus.metrics + statement: route() where resource.attributes["job"] == "kube-state-metrics" {{- end }} ## Configuration for Source Processor @@ -197,29 +182,6 @@ source: sumologic_schema: add_cloud_namespace: false -{{- if eq .Values.sumologic.metrics.sourceType "http" }} -transform/prepare_routing: - error_mode: ignore - metric_statements: - - context: metric - statements: - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.apiserver") where resource.attributes["job"] == "apiserver" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.control-plane.coredns") where resource.attributes["job"] == "coredns" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.control-plane.kube-etcd") where resource.attributes["job"] == "kube-etcd" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.controller-manager") where resource.attributes["job"] == "kubelet" and IsMatch(name, "^cloudprovider_.*") - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.kubelet") where resource.attributes["job"] == "kubelet" and not IsMatch(name, "^cloudprovider_.*") - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.node") where resource.attributes["job"] == "node-exporter" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.scheduler") where resource.attributes["job"] == "kube-scheduler" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.state") where resource.attributes["job"] == "kube-state-metrics" -{{ else }} -transform/drop_routing_attribute: - error_mode: ignore - metric_statements: - - context: resource - statements: - - delete_key(attributes, "http_listener_v2_path") -{{- end }} - transform/remove_name: error_mode: ignore metric_statements: diff --git a/tests/helm/metrics_test.go b/tests/helm/metrics_test.go index 8456507a13..e04869558f 100644 --- a/tests/helm/metrics_test.go +++ b/tests/helm/metrics_test.go @@ -137,7 +137,6 @@ func TestMetadataMetricsOtelConfigExtraProcessors(t *testing.T) { "transform/remove_name", "filter/drop_unnecessary_metrics", "batch", - "transform/prepare_routing", "routing", } @@ -182,10 +181,8 @@ sumologic: assert.Len(t, otelConfig.Exporters.Rest, 0) assert.NotContains(t, otelConfig.Processors, "routing") assert.NotContains(t, otelConfig.Processors, "transform/prepare_routing") - assert.Contains(t, otelConfig.Processors, "transform/drop_routing_attribute") assert.NotContains(t, otelConfig.Service.Pipelines.Metrics.Processors, "routing") assert.NotContains(t, otelConfig.Service.Pipelines.Metrics.Processors, "transform/prepare_routing") - assert.Contains(t, otelConfig.Service.Pipelines.Metrics.Processors, "transform/drop_routing_attribute") assert.Equal(t, otelConfig.Service.Pipelines.Metrics.Exporters, []string{"sumologic/default"}) } diff --git a/tests/helm/testdata/goldenfile/metadata_metrics_otc/additional_endpoints.output.yaml b/tests/helm/testdata/goldenfile/metadata_metrics_otc/additional_endpoints.output.yaml index a5d378640e..3b4509a825 100644 --- a/tests/helm/testdata/goldenfile/metadata_metrics_otc/additional_endpoints.output.yaml +++ b/tests/helm/testdata/goldenfile/metadata_metrics_otc/additional_endpoints.output.yaml @@ -202,69 +202,40 @@ data: - action: delete key: k8s.pod.pod_name routing: - attribute_source: resource default_exporters: - sumologic/default - drop_resource_routing_attribute: true - from_attribute: http_listener_v2_path + error_mode: ignore table: - exporters: - sumologic/apiserver - value: /prometheus.metrics.apiserver + statement: route() where resource.attributes["job"] == "apiserver" - exporters: - sumologic/control_plane - value: /prometheus.metrics.control-plane.coredns + statement: route() where resource.attributes["job"] == "coredns" - exporters: - sumologic/control_plane - value: /prometheus.metrics.control-plane.kube-etcd + statement: route() where resource.attributes["job"] == "kube-etcd" - exporters: - sumologic/controller - value: /prometheus.metrics.controller-manager + statement: route() where resource.attributes["job"] == "kube-controller-manager" - exporters: - sumologic/kubelet - value: /prometheus.metrics.kubelet + statement: route() where resource.attributes["job"] == "kubelet" - exporters: - sumologic/node - value: /prometheus.metrics.node + statement: route() where resource.attributes["job"] == "node-exporter" - exporters: - sumologic/scheduler - value: /prometheus.metrics.scheduler + statement: route() where resource.attributes["job"] == "kube-scheduler" - exporters: - sumologic/state - value: /prometheus.metrics.state - - exporters: - - sumologic/default - value: /prometheus.metrics.applications.custom - - exporters: - - sumologic/default - value: /prometheus.metrics + statement: route() where resource.attributes["job"] == "kube-state-metrics" source: collector: kubernetes exclude: k8s.namespace.name: "" sumologic_schema: add_cloud_namespace: false - transform/prepare_routing: - error_mode: ignore - metric_statements: - - context: metric - statements: - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.apiserver") - where resource.attributes["job"] == "apiserver" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.control-plane.coredns") - where resource.attributes["job"] == "coredns" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.control-plane.kube-etcd") - where resource.attributes["job"] == "kube-etcd" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.controller-manager") - where resource.attributes["job"] == "kubelet" and IsMatch(name, "^cloudprovider_.*") - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.kubelet") - where resource.attributes["job"] == "kubelet" and not IsMatch(name, "^cloudprovider_.*") - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.node") - where resource.attributes["job"] == "node-exporter" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.scheduler") - where resource.attributes["job"] == "kube-scheduler" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.state") - where resource.attributes["job"] == "kube-state-metrics" transform/remove_name: error_mode: ignore metric_statements: @@ -294,7 +265,6 @@ data: write_timeout = "30s" service_address = ":9888" data_format = "prometheusremotewrite" - path_tag = true paths = [ "/prometheus.metrics", "/prometheus.metrics.custom", @@ -331,7 +301,6 @@ data: - transform/remove_name - filter/drop_unnecessary_metrics - batch - - transform/prepare_routing - routing receivers: - telegraf diff --git a/tests/helm/testdata/goldenfile/metadata_metrics_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/metadata_metrics_otc/basic.output.yaml index b620906d3e..ef38dc4eb4 100644 --- a/tests/helm/testdata/goldenfile/metadata_metrics_otc/basic.output.yaml +++ b/tests/helm/testdata/goldenfile/metadata_metrics_otc/basic.output.yaml @@ -202,69 +202,40 @@ data: - action: delete key: k8s.pod.pod_name routing: - attribute_source: resource default_exporters: - sumologic/default - drop_resource_routing_attribute: true - from_attribute: http_listener_v2_path + error_mode: ignore table: - exporters: - sumologic/apiserver - value: /prometheus.metrics.apiserver + statement: route() where resource.attributes["job"] == "apiserver" - exporters: - sumologic/control_plane - value: /prometheus.metrics.control-plane.coredns + statement: route() where resource.attributes["job"] == "coredns" - exporters: - sumologic/control_plane - value: /prometheus.metrics.control-plane.kube-etcd + statement: route() where resource.attributes["job"] == "kube-etcd" - exporters: - sumologic/controller - value: /prometheus.metrics.controller-manager + statement: route() where resource.attributes["job"] == "kube-controller-manager" - exporters: - sumologic/kubelet - value: /prometheus.metrics.kubelet + statement: route() where resource.attributes["job"] == "kubelet" - exporters: - sumologic/node - value: /prometheus.metrics.node + statement: route() where resource.attributes["job"] == "node-exporter" - exporters: - sumologic/scheduler - value: /prometheus.metrics.scheduler + statement: route() where resource.attributes["job"] == "kube-scheduler" - exporters: - sumologic/state - value: /prometheus.metrics.state - - exporters: - - sumologic/default - value: /prometheus.metrics.applications.custom - - exporters: - - sumologic/default - value: /prometheus.metrics + statement: route() where resource.attributes["job"] == "kube-state-metrics" source: collector: kubernetes exclude: k8s.namespace.name: "" sumologic_schema: add_cloud_namespace: false - transform/prepare_routing: - error_mode: ignore - metric_statements: - - context: metric - statements: - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.apiserver") - where resource.attributes["job"] == "apiserver" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.control-plane.coredns") - where resource.attributes["job"] == "coredns" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.control-plane.kube-etcd") - where resource.attributes["job"] == "kube-etcd" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.controller-manager") - where resource.attributes["job"] == "kubelet" and IsMatch(name, "^cloudprovider_.*") - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.kubelet") - where resource.attributes["job"] == "kubelet" and not IsMatch(name, "^cloudprovider_.*") - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.node") - where resource.attributes["job"] == "node-exporter" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.scheduler") - where resource.attributes["job"] == "kube-scheduler" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.state") - where resource.attributes["job"] == "kube-state-metrics" transform/remove_name: error_mode: ignore metric_statements: @@ -294,7 +265,6 @@ data: write_timeout = "30s" service_address = ":9888" data_format = "prometheusremotewrite" - path_tag = true paths = [ "/prometheus.metrics" ] @@ -329,7 +299,6 @@ data: - transform/remove_name - filter/drop_unnecessary_metrics - batch - - transform/prepare_routing - routing receivers: - telegraf diff --git a/tests/helm/testdata/goldenfile/metadata_metrics_otc/custom.output.yaml b/tests/helm/testdata/goldenfile/metadata_metrics_otc/custom.output.yaml index 731f50690f..85df3500e1 100644 --- a/tests/helm/testdata/goldenfile/metadata_metrics_otc/custom.output.yaml +++ b/tests/helm/testdata/goldenfile/metadata_metrics_otc/custom.output.yaml @@ -137,12 +137,6 @@ data: k8s.namespace.name: "" sumologic_schema: add_cloud_namespace: false - transform/drop_routing_attribute: - error_mode: ignore - metric_statements: - - context: resource - statements: - - delete_key(attributes, "http_listener_v2_path") transform/remove_name: error_mode: ignore metric_statements: @@ -172,7 +166,6 @@ data: write_timeout = "30s" service_address = ":9888" data_format = "prometheusremotewrite" - path_tag = true paths = [ "/prometheus.metrics" ] @@ -200,7 +193,6 @@ data: - transform/remove_name - filter/drop_unnecessary_metrics - batch - - transform/drop_routing_attribute receivers: - telegraf - otlp diff --git a/tests/helm/testdata/goldenfile/metadata_metrics_otc/filtered_app_metrics.output.yaml b/tests/helm/testdata/goldenfile/metadata_metrics_otc/filtered_app_metrics.output.yaml index 02bc98530b..e35fd881f0 100644 --- a/tests/helm/testdata/goldenfile/metadata_metrics_otc/filtered_app_metrics.output.yaml +++ b/tests/helm/testdata/goldenfile/metadata_metrics_otc/filtered_app_metrics.output.yaml @@ -226,69 +226,40 @@ data: - action: delete key: k8s.pod.pod_name routing: - attribute_source: resource default_exporters: - sumologic/default - drop_resource_routing_attribute: true - from_attribute: http_listener_v2_path + error_mode: ignore table: - exporters: - sumologic/apiserver - value: /prometheus.metrics.apiserver + statement: route() where resource.attributes["job"] == "apiserver" - exporters: - sumologic/control_plane - value: /prometheus.metrics.control-plane.coredns + statement: route() where resource.attributes["job"] == "coredns" - exporters: - sumologic/control_plane - value: /prometheus.metrics.control-plane.kube-etcd + statement: route() where resource.attributes["job"] == "kube-etcd" - exporters: - sumologic/controller - value: /prometheus.metrics.controller-manager + statement: route() where resource.attributes["job"] == "kube-controller-manager" - exporters: - sumologic/kubelet - value: /prometheus.metrics.kubelet + statement: route() where resource.attributes["job"] == "kubelet" - exporters: - sumologic/node - value: /prometheus.metrics.node + statement: route() where resource.attributes["job"] == "node-exporter" - exporters: - sumologic/scheduler - value: /prometheus.metrics.scheduler + statement: route() where resource.attributes["job"] == "kube-scheduler" - exporters: - sumologic/state - value: /prometheus.metrics.state - - exporters: - - sumologic/default - value: /prometheus.metrics.applications.custom - - exporters: - - sumologic/default - value: /prometheus.metrics + statement: route() where resource.attributes["job"] == "kube-state-metrics" source: collector: kubernetes exclude: k8s.namespace.name: "" sumologic_schema: add_cloud_namespace: false - transform/prepare_routing: - error_mode: ignore - metric_statements: - - context: metric - statements: - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.apiserver") - where resource.attributes["job"] == "apiserver" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.control-plane.coredns") - where resource.attributes["job"] == "coredns" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.control-plane.kube-etcd") - where resource.attributes["job"] == "kube-etcd" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.controller-manager") - where resource.attributes["job"] == "kubelet" and IsMatch(name, "^cloudprovider_.*") - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.kubelet") - where resource.attributes["job"] == "kubelet" and not IsMatch(name, "^cloudprovider_.*") - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.node") - where resource.attributes["job"] == "node-exporter" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.scheduler") - where resource.attributes["job"] == "kube-scheduler" - - set(resource.attributes["http_listener_v2_path"], "/prometheus.metrics.state") - where resource.attributes["job"] == "kube-state-metrics" transform/remove_name: error_mode: ignore metric_statements: @@ -318,7 +289,6 @@ data: write_timeout = "30s" service_address = ":9888" data_format = "prometheusremotewrite" - path_tag = true paths = [ "/prometheus.metrics" ] @@ -354,7 +324,6 @@ data: - filter/drop_unnecessary_metrics - filter/app_metrics - batch - - transform/prepare_routing - routing receivers: - telegraf