fix(metrics): drop stale datapoints

SumoLogic · Oct 13, 2023 · 70af83f · 70af83f
1 parent 1f73a0d
commit 70af83f
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 0 deletions.
diff --git a/.changelog/3318.fixed.txt b/.changelog/3318.fixed.txt
@@ -0,0 +1 @@
+fix(metrics): drop stale datapoints
diff --git a/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml
@@ -27,6 +27,12 @@ processors:
     send_batch_size: 1000
     timeout: 1s
 
+  # staleness markers may simply indicate targets being moved between collector Pods, so they do more harm than good
+  filter/drop_stale_datapoints:
+    metrics:
+      datapoint:
+      - 'flags == FLAG_NO_RECORDED_VALUE'
+
   transform/drop_unnecessary_attributes:
     error_mode: ignore
     metric_statements:
@@ -203,6 +209,7 @@ service:
       exporters: [otlphttp]
       processors:
         - batch
+        - filter/drop_stale_datapoints
 {{- if .Values.sumologic.metrics.dropHistogramBuckets }}
         - transform/extract_sum_count_from_histograms
 {{- end }}

diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml
@@ -86,6 +86,10 @@ spec:
         send_batch_max_size: 2000
         send_batch_size: 1000
         timeout: 1s
+      filter/drop_stale_datapoints:
+        metrics:
+          datapoint:
+          - flags == FLAG_NO_RECORDED_VALUE
       transform/drop_unnecessary_attributes:
         error_mode: ignore
         metric_statements:
@@ -242,6 +246,7 @@ spec:
           - otlphttp
           processors:
           - batch
+          - filter/drop_stale_datapoints
           - transform/extract_sum_count_from_histograms
           - transform/drop_unnecessary_attributes
           receivers:

diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml
@@ -105,6 +105,10 @@ spec:
         send_batch_max_size: 2000
         send_batch_size: 5000
         timeout: 1s
+      filter/drop_stale_datapoints:
+        metrics:
+          datapoint:
+          - flags == FLAG_NO_RECORDED_VALUE
       transform/drop_unnecessary_attributes:
         error_mode: ignore
         metric_statements:
@@ -136,6 +140,7 @@ spec:
           - otlphttp
           processors:
           - batch
+          - filter/drop_stale_datapoints
           - transform/drop_unnecessary_attributes
           receivers:
           - prometheus