From 70af83f69e2b856d4173fdfd3e11739de47b8a43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?=
 <mail+sumo@mikolajswiatek.com>
Date: Thu, 12 Oct 2023 16:34:20 +0200
Subject: [PATCH] fix(metrics): drop stale datapoints

---
 .changelog/3318.fixed.txt                                  | 1 +
 .../sumologic/conf/metrics/collector/otelcol/config.yaml   | 7 +++++++
 .../goldenfile/metrics_collector_otc/basic.output.yaml     | 5 +++++
 .../goldenfile/metrics_collector_otc/custom.output.yaml    | 5 +++++
 4 files changed, 18 insertions(+)
 create mode 100644 .changelog/3318.fixed.txt

diff --git a/.changelog/3318.fixed.txt b/.changelog/3318.fixed.txt
new file mode 100644
index 0000000000..b847aebbd5
--- /dev/null
+++ b/.changelog/3318.fixed.txt
@@ -0,0 +1 @@
+fix(metrics): drop stale datapoints
\ No newline at end of file
diff --git a/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml
index eadb184e3c..fef54fb6ab 100644
--- a/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml
+++ b/deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml
@@ -27,6 +27,12 @@ processors:
     send_batch_size: 1000
     timeout: 1s
 
+  # staleness markers may simply indicate targets being moved between collector Pods, so they do more harm than good
+  filter/drop_stale_datapoints:
+    metrics:
+      datapoint:
+      - 'flags == FLAG_NO_RECORDED_VALUE'
+
   transform/drop_unnecessary_attributes:
     error_mode: ignore
     metric_statements:
@@ -203,6 +209,7 @@ service:
       exporters: [otlphttp]
       processors:
         - batch
+        - filter/drop_stale_datapoints
 {{- if .Values.sumologic.metrics.dropHistogramBuckets }}
         - transform/extract_sum_count_from_histograms
 {{- end }}
diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml
index 70ef273500..83d6c686f0 100644
--- a/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml
+++ b/tests/helm/testdata/goldenfile/metrics_collector_otc/basic.output.yaml
@@ -86,6 +86,10 @@ spec:
         send_batch_max_size: 2000
         send_batch_size: 1000
         timeout: 1s
+      filter/drop_stale_datapoints:
+        metrics:
+          datapoint:
+          - flags == FLAG_NO_RECORDED_VALUE
       transform/drop_unnecessary_attributes:
         error_mode: ignore
         metric_statements:
@@ -242,6 +246,7 @@ spec:
           - otlphttp
           processors:
           - batch
+          - filter/drop_stale_datapoints
           - transform/extract_sum_count_from_histograms
           - transform/drop_unnecessary_attributes
           receivers:
diff --git a/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml b/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml
index 9d74926572..baeff887f7 100644
--- a/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml
+++ b/tests/helm/testdata/goldenfile/metrics_collector_otc/custom.output.yaml
@@ -105,6 +105,10 @@ spec:
         send_batch_max_size: 2000
         send_batch_size: 5000
         timeout: 1s
+      filter/drop_stale_datapoints:
+        metrics:
+          datapoint:
+          - flags == FLAG_NO_RECORDED_VALUE
       transform/drop_unnecessary_attributes:
         error_mode: ignore
         metric_statements:
@@ -136,6 +140,7 @@ spec:
           - otlphttp
           processors:
           - batch
+          - filter/drop_stale_datapoints
           - transform/drop_unnecessary_attributes
           receivers:
           - prometheus