From abb815d39fb5495ab6ddf5a1c6800261993d3d49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?=
 <mail+sumo@mikolajswiatek.com>
Date: Fri, 13 Oct 2023 18:15:28 +0200
Subject: [PATCH] fix(metrics): kube-state-metrics pod metadata

---
 .changelog/3323.fixed.txt         |  1 +
 deploy/helm/sumologic/values.yaml |  9 ++++---
 tests/integration/features.go     | 45 ++++++++++++++++++++++++++++++-
 3 files changed, 50 insertions(+), 5 deletions(-)
 create mode 100644 .changelog/3323.fixed.txt

diff --git a/.changelog/3323.fixed.txt b/.changelog/3323.fixed.txt
new file mode 100644
index 0000000000..577ae448d6
--- /dev/null
+++ b/.changelog/3323.fixed.txt
@@ -0,0 +1 @@
+fix(metrics): kube-state-metrics pod metadata
\ No newline at end of file
diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
index 973b7d52df..0af569eccc 100644
--- a/deploy/helm/sumologic/values.yaml
+++ b/deploy/helm/sumologic/values.yaml
@@ -1062,16 +1062,17 @@ kube-prometheus-stack:
             sourceLabels: [__name__]
           ## Drop unnecessary labels Prometheus adds to these metrics
           ## We don't want container=kube-state-metrics on everything
+          ## But we do want to keep these on pod metrics, which we check for via the `uid` attribute
           - action: labeldrop
             regex: service
           - action: replace
-            sourceLabels: [container]
-            regex: kube-state-metrics
+            sourceLabels: [container, uid]
+            regex: kube-state-metrics;
             targetLabel: container
             replacement: ""
           - action: replace
-            sourceLabels: [pod]
-            regex: ".*kube-state-metrics.*"
+            sourceLabels: [pod, uid]
+            regex: ".*kube-state-metrics.*;"
             targetLabel: pod
             replacement: ""
           - action: labelmap
diff --git a/tests/integration/features.go b/tests/integration/features.go
index c37ec70d7e..519f3617d3 100644
--- a/tests/integration/features.go
+++ b/tests/integration/features.go
@@ -103,7 +103,7 @@ func GetMetricsFeature(expectedMetrics []string, metricsCollector MetricsCollect
 				return stepfuncs.WaitUntilExpectedMetricLabelsPresent(metricFilters, expectedLabels, waitDuration, tickDuration)(ctx, t, envConf)
 			},
 		).
-		Assess("expected labels are present for kube-state-metrics",
+		Assess("expected labels are present for non-pod kube-state-metrics",
 			func(ctx context.Context, t *testing.T, envConf *envconf.Config) context.Context {
 				metricFilters := receivermock.MetadataFilters{
 					"__name__":   "kube_deployment_spec_replicas",
@@ -126,6 +126,49 @@ func GetMetricsFeature(expectedMetrics []string, metricsCollector MetricsCollect
 				return stepfuncs.WaitUntilExpectedMetricLabelsPresent(metricFilters, expectedLabels, waitDuration, tickDuration)(ctx, t, envConf)
 			},
 		).
+		Assess("expected labels are present for pod kube-state-metrics",
+			func(ctx context.Context, t *testing.T, envConf *envconf.Config) context.Context {
+				releaseName := ctxopts.HelmRelease(ctx)
+				deployment := fmt.Sprintf("%s-kube-state-metrics", releaseName)
+				metricFilters := receivermock.MetadataFilters{
+					"__name__":   "kube_pod_status_phase",
+					"phase":      "Running",
+					"deployment": deployment,
+				}
+				namespace := ctxopts.Namespace(ctx)
+				expectedLabels := receivermock.Labels{
+					"cluster":                                "kubernetes",
+					"_origin":                                "kubernetes",
+					"container":                              "kube-state-metrics",
+					"deployment":                             deployment,
+					"endpoint":                               "http",
+					"job":                                    "kube-state-metrics",
+					"namespace":                              namespace,
+					"node":                                   internal.NodeNameRegex,
+					"phase":                                  "Running",
+					"pod_labels_app.kubernetes.io/component": "metrics",
+					"pod_labels_app.kubernetes.io/instance":  releaseName,
+					"pod_labels_app.kubernetes.io/managed-by": "Helm",
+					"pod_labels_app.kubernetes.io/name":       "kube-state-metrics",
+					"pod_labels_app.kubernetes.io/part-of":    "kube-state-metrics",
+					"pod_labels_app.kubernetes.io/version":    "\\d+\\.\\d+\\.\\d+",
+					"pod_labels_app.kubernetes.io/chart":      "kube-state-metrics-\\d+\\.\\d+\\.\\d+",
+					"pod_labels_release":                      releaseName,
+					"pod_labels_pod-template-hash":            ".+",
+					"pod_labels_service":                      "receiver-mock",
+					"pod":                                     fmt.Sprintf("%s-.+", deployment),
+					"replicaset":                              fmt.Sprintf("%s-.+", deployment),
+					"service":                                 deployment,
+					"service_discovery_pod":                   fmt.Sprintf("%s-.+", deployment),
+					"uid":                                     ".+",
+				}
+				expectedLabels = addCollectorSpecificMetricLabels(expectedLabels, releaseName, namespace, metricsCollector)
+				// drop some unnecessary labels
+				delete(expectedLabels, "prometheus_service")
+
+				return stepfuncs.WaitUntilExpectedMetricLabelsPresent(metricFilters, expectedLabels, waitDuration, tickDuration)(ctx, t, envConf)
+			},
+		).
 		Feature()
 }