From beb29a78d420c18e7e14447a01f809eb0e403fa4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?=
 <mail+sumo@mikolajswiatek.com>
Date: Thu, 12 Oct 2023 16:34:20 +0200
Subject: [PATCH] fix(metrics): drop stale datapoints

(cherry picked from commit 70af83f69e2b856d4173fdfd3e11739de47b8a43)
---
 .changelog/3318.fixed.txt                                  | 1 +
 .../sumologic/conf/metrics/collector/otelcol/config.yaml   | 7 +++++++
 .../goldenfile/metrics_collector_otc/basic.output.yaml     | 7 +++++++
 .../goldenfile/metrics_collector_otc/custom.output.yaml    | 7 +++++++
 4 files changed, 22 insertions(+)
 create mode 100644 .changelog/3318.fixed.txt

diff --git a/.changelog/3318.fixed.txt b/.changelog/3318.fixed.txt
new file mode 100644
index 0000000000..b847aebbd5
--- /dev/null
+++ b/.changelog/3318.fixed.txt
@@ -0,0 +1 @@
+fix(metrics): drop stale datapoints
\ No newline at end of file
diff --git a/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml
index 3c2eb3738a..15ccd89cc4 100644
--- a/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml
+++ b/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml
@@ -27,6 +27,12 @@ processors:
     send_batch_size: 1000
     timeout: 1s
 
+  # staleness markers may simply indicate targets being moved between collector Pods, so they do more harm than good
+  filter/drop_stale_datapoints:
+    metrics:
+      datapoint:
+      - 'flags == FLAG_NO_RECORDED_VALUE'
+
   transform/drop_unnecessary_attributes:
     error_mode: ignore
     metric_statements:
@@ -217,6 +223,7 @@ service:
       exporters: [otlphttp]
       processors:
         - batch
+        - filter/drop_stale_datapoints
 {{- if .Values.sumologic.metrics.dropHistogramBuckets }}
         - transform/extract_sum_count_from_histograms
 {{- end }}
diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml
index bcebf2c2ad..9b4bcc0dec 100644
--- a/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml
+++ b/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml
@@ -91,6 +91,12 @@ spec:
         send_batch_size: 1000
         timeout: 1s
 
+      # staleness markers may simply indicate targets being moved between collector Pods, so they do more harm than good
+      filter/drop_stale_datapoints:
+        metrics:
+          datapoint:
+          - 'flags == FLAG_NO_RECORDED_VALUE'
+
       transform/drop_unnecessary_attributes:
         error_mode: ignore
         metric_statements:
@@ -263,6 +269,7 @@ spec:
           exporters: [otlphttp]
           processors:
             - batch
+            - filter/drop_stale_datapoints
             - transform/extract_sum_count_from_histograms
             - filter/drop_unnecessary_metrics
             - transform/drop_unnecessary_attributes
diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml
index 8e060ba16d..ab625721a9 100644
--- a/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml
+++ b/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml
@@ -110,6 +110,12 @@ spec:
         send_batch_size: 1000
         timeout: 1s
 
+      # staleness markers may simply indicate targets being moved between collector Pods, so they do more harm than good
+      filter/drop_stale_datapoints:
+        metrics:
+          datapoint:
+          - 'flags == FLAG_NO_RECORDED_VALUE'
+
       transform/drop_unnecessary_attributes:
         error_mode: ignore
         metric_statements:
@@ -1071,6 +1077,7 @@ spec:
           exporters: [otlphttp]
           processors:
             - batch
+            - filter/drop_stale_datapoints
             - filter/drop_unnecessary_metrics
             - transform/drop_unnecessary_attributes
             - filter/app_metrics