Fix recording rule errors the label values change (#6)

This fixes the following problem: You define an SLO with label mylabel=original. Let that run for a while. Then you change the label to mylabel=changed. This will cause the slo:sli_error:ratio_rate<window> to fail for the duration of the window with the error vector contains metrics with the same labelset after applying rule labels. To fix this problem we essentially discard the labels from the query. They are then re-applied to the resulting recording rule. Note that this uses sub-queries so you need to be running a version of Prometheus that supports that.
kolonialno · Aug 31, 2023 · f813004 · f813004
1 parent cb76276
commit f813004
Show file tree

Hide file tree

Showing 30 changed files with 140 additions and 140 deletions.
diff --git a/examples/_gen/getting-started.yml b/examples/_gen/getting-started.yml
@@ -106,9 +106,9 @@ groups:
       tier: "2"
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"})[30d:])
     labels:
       cmd: examplesgen.sh
       owner: myteam

diff --git a/examples/_gen/home-wifi.yml b/examples/_gen/home-wifi.yml
@@ -106,9 +106,9 @@ groups:
       sloth_window: 3d
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-good-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="good-wifi-client-satisfaction"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-good-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="good-wifi-client-satisfaction"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-good-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="good-wifi-client-satisfaction"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-good-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="good-wifi-client-satisfaction"})[30d:])
     labels:
       cluster: valhalla
       cmd: examplesgen.sh
@@ -347,9 +347,9 @@ groups:
       sloth_window: 3d
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-risk-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="risk-wifi-client-satisfaction"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-risk-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="risk-wifi-client-satisfaction"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-risk-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="risk-wifi-client-satisfaction"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-risk-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="risk-wifi-client-satisfaction"})[30d:])
     labels:
       cluster: valhalla
       cmd: examplesgen.sh

diff --git a/examples/_gen/k8s-getting-started.yml b/examples/_gen/k8s-getting-started.yml
@@ -115,9 +115,9 @@ spec:
         tier: "2"
       record: slo:sli_error:ratio_rate3d
     - expr: |
-        sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"}[30d])
-        / ignoring (sloth_window)
-        count_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"}[30d])
+        sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"})[30d:])
+        /
+        count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"})[30d:])
       labels:
         cmd: examplesgen.sh
         owner: myteam

diff --git a/examples/_gen/k8s-home-wifi.yml b/examples/_gen/k8s-home-wifi.yml
@@ -118,9 +118,9 @@ spec:
         sloth_window: 3d
       record: slo:sli_error:ratio_rate3d
     - expr: |
-        sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-good-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="good-wifi-client-satisfaction"}[30d])
-        / ignoring (sloth_window)
-        count_over_time(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-good-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="good-wifi-client-satisfaction"}[30d])
+        sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-good-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="good-wifi-client-satisfaction"})[30d:])
+        /
+        count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-good-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="good-wifi-client-satisfaction"})[30d:])
       labels:
         cluster: valhalla
         cmd: examplesgen.sh
@@ -359,9 +359,9 @@ spec:
         sloth_window: 3d
       record: slo:sli_error:ratio_rate3d
     - expr: |
-        sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-risk-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="risk-wifi-client-satisfaction"}[30d])
-        / ignoring (sloth_window)
-        count_over_time(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-risk-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="risk-wifi-client-satisfaction"}[30d])
+        sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-risk-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="risk-wifi-client-satisfaction"})[30d:])
+        /
+        count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-risk-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="risk-wifi-client-satisfaction"})[30d:])
       labels:
         cluster: valhalla
         cmd: examplesgen.sh

diff --git a/examples/_gen/k8s-multifile.yml b/examples/_gen/k8s-multifile.yml
@@ -115,9 +115,9 @@ spec:
         tier: "2"
       record: slo:sli_error:ratio_rate3d
     - expr: |
-        sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"}[30d])
-        / ignoring (sloth_window)
-        count_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"}[30d])
+        sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"})[30d:])
+        /
+        count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"})[30d:])
       labels:
         cmd: examplesgen.sh
         owner: myteam
@@ -374,9 +374,9 @@ spec:
         tier: "1"
       record: slo:sli_error:ratio_rate3d
     - expr: |
-        sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice2-requests-availability", sloth_service="myservice2", sloth_slo="requests-availability"}[30d])
-        / ignoring (sloth_window)
-        count_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice2-requests-availability", sloth_service="myservice2", sloth_slo="requests-availability"}[30d])
+        sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice2-requests-availability", sloth_service="myservice2", sloth_slo="requests-availability"})[30d:])
+        /
+        count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice2-requests-availability", sloth_service="myservice2", sloth_slo="requests-availability"})[30d:])
       labels:
         cmd: examplesgen.sh
         owner: myteam2

diff --git a/examples/_gen/kubernetes-apiserver.yml b/examples/_gen/kubernetes-apiserver.yml
@@ -106,9 +106,9 @@ groups:
       sloth_window: 3d
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-availability", sloth_service="k8s-apiserver", sloth_slo="requests-availability"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-availability", sloth_service="k8s-apiserver", sloth_slo="requests-availability"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-availability", sloth_service="k8s-apiserver", sloth_slo="requests-availability"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-availability", sloth_service="k8s-apiserver", sloth_slo="requests-availability"})[30d:])
     labels:
       category: availability
       cluster: valhalla
@@ -386,9 +386,9 @@ groups:
       sloth_window: 3d
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-latency", sloth_service="k8s-apiserver", sloth_slo="requests-latency"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-latency", sloth_service="k8s-apiserver", sloth_slo="requests-latency"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-latency", sloth_service="k8s-apiserver", sloth_slo="requests-latency"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-latency", sloth_service="k8s-apiserver", sloth_slo="requests-latency"})[30d:])
     labels:
       category: latency
       cluster: valhalla

diff --git a/examples/_gen/multifile.yml b/examples/_gen/multifile.yml
@@ -106,9 +106,9 @@ groups:
       tier: "2"
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"})[30d:])
     labels:
       cmd: examplesgen.sh
       owner: myteam
@@ -355,9 +355,9 @@ groups:
       tier: "1"
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice2-requests-availability", sloth_service="myservice2", sloth_slo="requests-availability"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice2-requests-availability", sloth_service="myservice2", sloth_slo="requests-availability"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice2-requests-availability", sloth_service="myservice2", sloth_slo="requests-availability"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice2-requests-availability", sloth_service="myservice2", sloth_slo="requests-availability"})[30d:])
     labels:
       cmd: examplesgen.sh
       owner: myteam2

diff --git a/examples/_gen/no-alerts.yml b/examples/_gen/no-alerts.yml
@@ -134,9 +134,9 @@ groups:
       sloth_window: 3d
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="myapp-http-availability", sloth_service="myapp", sloth_slo="http-availability"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="myapp-http-availability", sloth_service="myapp", sloth_slo="http-availability"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myapp-http-availability", sloth_service="myapp", sloth_slo="http-availability"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myapp-http-availability", sloth_service="myapp", sloth_slo="http-availability"})[30d:])
     labels:
       cmd: examplesgen.sh
       owner: myteam

diff --git a/examples/_gen/openslo-getting-started.yml b/examples/_gen/openslo-getting-started.yml
@@ -141,9 +141,9 @@ groups:
       sloth_window: 3d
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="my-service-sloth-slo-my-service-0", sloth_service="my-service", sloth_slo="sloth-slo-my-service-0"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="my-service-sloth-slo-my-service-0", sloth_service="my-service", sloth_slo="sloth-slo-my-service-0"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="my-service-sloth-slo-my-service-0", sloth_service="my-service", sloth_slo="sloth-slo-my-service-0"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="my-service-sloth-slo-my-service-0", sloth_service="my-service", sloth_slo="sloth-slo-my-service-0"})[30d:])
     labels:
       cmd: examplesgen.sh
       sloth_id: my-service-sloth-slo-my-service-0

diff --git a/examples/_gen/openslo-kubernetes-apiserver.yml b/examples/_gen/openslo-kubernetes-apiserver.yml
@@ -141,9 +141,9 @@ groups:
       sloth_window: 3d
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-availability-openslo-0", sloth_service="k8s-apiserver", sloth_slo="requests-availability-openslo-0"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-availability-openslo-0", sloth_service="k8s-apiserver", sloth_slo="requests-availability-openslo-0"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-availability-openslo-0", sloth_service="k8s-apiserver", sloth_slo="requests-availability-openslo-0"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-availability-openslo-0", sloth_service="k8s-apiserver", sloth_slo="requests-availability-openslo-0"})[30d:])
     labels:
       cmd: examplesgen.sh
       sloth_id: k8s-apiserver-requests-availability-openslo-0
@@ -355,9 +355,9 @@ groups:
       sloth_window: 3d
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-latency-openslo-0", sloth_service="k8s-apiserver", sloth_slo="requests-latency-openslo-0"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-latency-openslo-0", sloth_service="k8s-apiserver", sloth_slo="requests-latency-openslo-0"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-latency-openslo-0", sloth_service="k8s-apiserver", sloth_slo="requests-latency-openslo-0"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-latency-openslo-0", sloth_service="k8s-apiserver", sloth_slo="requests-latency-openslo-0"})[30d:])
     labels:
       cmd: examplesgen.sh
       sloth_id: k8s-apiserver-requests-latency-openslo-0
@@ -563,9 +563,9 @@ groups:
       sloth_window: 3d
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-latency-openslo-1", sloth_service="k8s-apiserver", sloth_slo="requests-latency-openslo-1"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-latency-openslo-1", sloth_service="k8s-apiserver", sloth_slo="requests-latency-openslo-1"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-latency-openslo-1", sloth_service="k8s-apiserver", sloth_slo="requests-latency-openslo-1"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="k8s-apiserver-requests-latency-openslo-1", sloth_service="k8s-apiserver", sloth_slo="requests-latency-openslo-1"})[30d:])
     labels:
       cmd: examplesgen.sh
       sloth_id: k8s-apiserver-requests-latency-openslo-1

diff --git a/examples/_gen/plugin-getting-started.yml b/examples/_gen/plugin-getting-started.yml
@@ -113,9 +113,9 @@ groups:
       tier: "2"
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"})[30d:])
     labels:
       cmd: examplesgen.sh
       owner: myteam

diff --git a/examples/_gen/plugin-k8s-getting-started.yml b/examples/_gen/plugin-k8s-getting-started.yml
@@ -125,9 +125,9 @@ spec:
         tier: "2"
       record: slo:sli_error:ratio_rate3d
     - expr: |
-        sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"}[30d])
-        / ignoring (sloth_window)
-        count_over_time(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"}[30d])
+        sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"})[30d:])
+        /
+        count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="myservice-requests-availability", sloth_service="myservice", sloth_slo="requests-availability"})[30d:])
       labels:
         cmd: examplesgen.sh
         owner: myteam

diff --git a/examples/_gen/raw-home-wifi.yml b/examples/_gen/raw-home-wifi.yml
@@ -127,9 +127,9 @@ groups:
       sloth_window: 3d
   - record: slo:sli_error:ratio_rate30d
     expr: |
-      sum_over_time(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="wifi-client-satisfaction"}[30d])
-      / ignoring (sloth_window)
-      count_over_time(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="wifi-client-satisfaction"}[30d])
+      sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="wifi-client-satisfaction"})[30d:])
+      /
+      count_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id="home-wifi-wifi-client-satisfaction", sloth_service="home-wifi", sloth_slo="wifi-client-satisfaction"})[30d:])
     labels:
       cluster: valhalla
       cmd: examplesgen.sh

diff --git a/internal/app/generate/prometheus_test.go b/internal/app/generate/prometheus_test.go
@@ -226,7 +226,7 @@ func TestIntegrationAppServiceGenerate(t *testing.T) {
 								},
 								{
 									Record: "slo:sli_error:ratio_rate30d",
-									Expr:   "sum_over_time(slo:sli_error:ratio_rate5m{sloth_id=\"test-id\", sloth_service=\"test-svc\", sloth_slo=\"test-name\"}[30d])\n/ ignoring (sloth_window)\ncount_over_time(slo:sli_error:ratio_rate5m{sloth_id=\"test-id\", sloth_service=\"test-svc\", sloth_slo=\"test-name\"}[30d])\n",
+									Expr:   "sum_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id=\"test-id\", sloth_service=\"test-svc\", sloth_slo=\"test-name\"})[30d:])\n/\ncount_over_time(sum(slo:sli_error:ratio_rate5m{sloth_id=\"test-id\", sloth_service=\"test-svc\", sloth_slo=\"test-name\"})[30d:])\n",
 									Labels: map[string]string{
 										"test_label":    "label_1",
 										"extra_k1":      "extra_v1",

diff --git a/internal/prometheus/recording_rules.go b/internal/prometheus/recording_rules.go
@@ -153,9 +153,9 @@ func optimizedSLIRecordGenerator(slo SLO, window, shortWindow time.Duration) (*r
 	// that is 1 (thats why we can use `count`), giving use a correct ratio of ratios:
 	// - https://prometheus.io/docs/practices/rules/
 	// - https://math.stackexchange.com/questions/95909/why-is-an-average-of-an-average-usually-incorrect
-	const sliExprTplFmt = `sum_over_time({{.metric}}{{.filter}}[{{.window}}])
-/ ignoring ({{.windowKey}})
-count_over_time({{.metric}}{{.filter}}[{{.window}}])
+	const sliExprTplFmt = `sum_over_time(sum({{.metric}}{{.filter}})[{{.window}}:])
+/
+count_over_time(sum({{.metric}}{{.filter}})[{{.window}}:])
 `
 
 	if window == shortWindow {