fix(metrics): kube-state-metrics pod metadata

SumoLogic · Oct 14, 2023 · b5abe25 · b5abe25
1 parent 70af83f
commit b5abe25
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 5 deletions.
diff --git a/.changelog/3323.fixed.txt b/.changelog/3323.fixed.txt
@@ -0,0 +1 @@
+fix(metrics): kube-state-metrics pod metadata
diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -1062,16 +1062,17 @@ kube-prometheus-stack:
             sourceLabels: [__name__]
           ## Drop unnecessary labels Prometheus adds to these metrics
           ## We don't want container=kube-state-metrics on everything
+          ## But we do want to keep these on pod metrics, which we check for via the `uid` attribute
           - action: labeldrop
             regex: service
           - action: replace
-            sourceLabels: [container]
-            regex: kube-state-metrics
+            sourceLabels: [container, uid]
+            regex: kube-state-metrics;
             targetLabel: container
             replacement: ""
           - action: replace
-            sourceLabels: [pod]
-            regex: ".*kube-state-metrics.*"
+            sourceLabels: [pod, uid]
+            regex: ".*kube-state-metrics.*;"
             targetLabel: pod
             replacement: ""
           - action: labelmap

diff --git a/tests/integration/features.go b/tests/integration/features.go
@@ -103,7 +103,7 @@ func GetMetricsFeature(expectedMetrics []string, metricsCollector MetricsCollect
 				return stepfuncs.WaitUntilExpectedMetricLabelsPresent(metricFilters, expectedLabels, waitDuration, tickDuration)(ctx, t, envConf)
 			},
 		).
-		Assess("expected labels are present for kube-state-metrics",
+		Assess("expected labels are present for non-pod kube-state-metrics",
 			func(ctx context.Context, t *testing.T, envConf *envconf.Config) context.Context {
 				metricFilters := receivermock.MetadataFilters{
 					"__name__":   "kube_deployment_spec_replicas",
@@ -126,6 +126,48 @@ func GetMetricsFeature(expectedMetrics []string, metricsCollector MetricsCollect
 				return stepfuncs.WaitUntilExpectedMetricLabelsPresent(metricFilters, expectedLabels, waitDuration, tickDuration)(ctx, t, envConf)
 			},
 		).
+		Assess("expected labels are present for pod kube-state-metrics",
+			func(ctx context.Context, t *testing.T, envConf *envconf.Config) context.Context {
+				releaseName := ctxopts.HelmRelease(ctx)
+				deployment := fmt.Sprintf("%s-kube-state-metrics", releaseName)
+				metricFilters := receivermock.MetadataFilters{
+					"__name__":   "kube_pod_status_phase",
+					"phase":      "Running",
+					"deployment": deployment,
+				}
+				namespace := ctxopts.Namespace(ctx)
+				expectedLabels := receivermock.Labels{
+					"cluster":                                "kubernetes",
+					"_origin":                                "kubernetes",
+					"container":                              "kube-state-metrics",
+					"deployment":                             deployment,
+					"endpoint":                               "http",
+					"job":                                    "kube-state-metrics",
+					"namespace":                              namespace,
+					"node":                                   internal.NodeNameRegex,
+					"phase":                                  "Running",
+					"pod_labels_app.kubernetes.io/component": "metrics",
+					"pod_labels_app.kubernetes.io/instance":  releaseName,
+					"pod_labels_app.kubernetes.io/managed-by": "Helm",
+					"pod_labels_app.kubernetes.io/name":       "kube-state-metrics",
+					"pod_labels_app.kubernetes.io/part-of":    "kube-state-metrics",
+					"pod_labels_app.kubernetes.io/version":    "\\d+\\.\\d+\\.\\d+",
+					"pod_labels_helm.sh/chart":                "kube-state-metrics-\\d+\\.\\d+\\.\\d+",
+					"pod_labels_release":                      releaseName,
+					"pod_labels_pod-template-hash":            ".+",
+					"pod":                                     fmt.Sprintf("%s-.+", deployment),
+					"replicaset":                              fmt.Sprintf("%s-.+", deployment),
+					"service":                                 deployment,
+					"service_discovery_pod":                   fmt.Sprintf("%s-.+", deployment),
+					"uid":                                     ".+",
+				}
+				expectedLabels = addCollectorSpecificMetricLabels(expectedLabels, releaseName, namespace, metricsCollector)
+				// drop some unnecessary labels
+				delete(expectedLabels, "prometheus_service")
+
+				return stepfuncs.WaitUntilExpectedMetricLabelsPresent(metricFilters, expectedLabels, waitDuration, tickDuration)(ctx, t, envConf)
+			},
+		).
 		Feature()
 }