update monitoring-mixins

Signed-off-by: Weifeng Wang <[email protected]> update mimir-mixin Signed-off-by: Weifeng Wang <[email protected]> update pyroscope-mixin Signed-off-by: Weifeng Wang <[email protected]> update loki-mixin Signed-off-by: Weifeng Wang <[email protected]> update tempo-mixin Signed-off-by: Weifeng Wang <[email protected]> add tempo-mixin Signed-off-by: Weifeng Wang <[email protected]>
qclaogui · Apr 1, 2024 · 5bdb1b9 · 5bdb1b9
1 parent df0e697
commit 5bdb1b9
Show file tree

Hide file tree

Showing 75 changed files with 2,903 additions and 916 deletions.
diff --git a/compose.override.yaml b/compose.override.yaml
@@ -45,7 +45,7 @@ services:
     volumes:
       - ./monitoring-mixins/pyroscope-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/pyroscope-mixin
       - ./monitoring-mixins/loki-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/loki-mixin
-      # - ./monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
+      - ./monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
     environment:
       GF_LOG_LEVEL: ${GF_LOG_LEVEL:-warn}
       GF_DIAGNOSTICS_PROFILING_ENABLED: true
@@ -83,8 +83,8 @@ services:
       - ./monitoring-mixins/loki-mixin/deploy/loki-mixin-rules.yaml:/rules/loki-mixin-rules.yaml
       - ./monitoring-mixins/loki-mixin/deploy/loki-mixin-alerts.yaml:/rules/loki-mixin-alerts.yaml
       - ./monitoring-mixins/pyroscope-mixin/deploy/pyroscope-mixin-rules.yaml:/rules/pyroscope-mixin-rules.yaml
-      # - ./monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/rules/tempo-mixin-rules.yaml
-      # - ./monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/rules/tempo-mixin-alerts.yaml
+      - ./monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/rules/tempo-mixin-rules.yaml
+      - ./monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/rules/tempo-mixin-alerts.yaml
 
   # override compose.yaml included service pyroscope's labels and environment to enabled traces data collection
   pyroscope:

diff --git a/docker-compose/microservices-mode/traces/grafana.override.yaml b/docker-compose/microservices-mode/traces/grafana.override.yaml
@@ -4,5 +4,5 @@ services:
   grafana:
     volumes:
       - ../../../monitoring-mixins/loki-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/loki-mixin
-      # - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
+      - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
 
diff --git a/docker-compose/microservices-mode/traces/mimirtool.override.yaml b/docker-compose/microservices-mode/traces/mimirtool.override.yaml
@@ -5,5 +5,5 @@ services:
     volumes:
       - ../../../monitoring-mixins/loki-mixin/deploy/loki-mixin-rules.yaml:/rules/loki-mixin-rules.yaml
       - ../../../monitoring-mixins/loki-mixin/deploy/loki-mixin-alerts.yaml:/rules/loki-mixin-alerts.yaml
-      # - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/rules/tempo-mixin-rules.yaml
-      # - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/rules/tempo-mixin-alerts.yaml
+      - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/rules/tempo-mixin-rules.yaml
+      - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/rules/tempo-mixin-alerts.yaml
diff --git a/docker-compose/monolithic-mode/all-in-one/grafana.override.yaml b/docker-compose/monolithic-mode/all-in-one/grafana.override.yaml
@@ -12,7 +12,7 @@ services:
     volumes:
       - ../../../monitoring-mixins/pyroscope-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/pyroscope-mixin
       - ../../../monitoring-mixins/loki-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/loki-mixin
-      # - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
+      - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
     environment:
       GF_LOG_LEVEL: ${GF_LOG_LEVEL:-warn}
       GF_DIAGNOSTICS_PROFILING_ENABLED: true

diff --git a/docker-compose/monolithic-mode/all-in-one/mimirtool.override.yaml b/docker-compose/monolithic-mode/all-in-one/mimirtool.override.yaml
@@ -6,5 +6,5 @@ services:
       - ../../../monitoring-mixins/loki-mixin/deploy/loki-mixin-rules.yaml:/rules/loki-mixin-rules.yaml
       - ../../../monitoring-mixins/loki-mixin/deploy/loki-mixin-alerts.yaml:/rules/loki-mixin-alerts.yaml
       - ../../../monitoring-mixins/pyroscope-mixin/deploy/pyroscope-mixin-rules.yaml:/rules/pyroscope-mixin-rules.yaml
-      # - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/rules/tempo-mixin-rules.yaml
-      # - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/rules/tempo-mixin-alerts.yaml
+      - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/rules/tempo-mixin-rules.yaml
+      - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/rules/tempo-mixin-alerts.yaml
diff --git a/docker-compose/monolithic-mode/traces/grafana.override.yaml b/docker-compose/monolithic-mode/traces/grafana.override.yaml
@@ -4,5 +4,5 @@ services:
   grafana:
     volumes:
       - ../../../monitoring-mixins/loki-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/loki-mixin
-      # - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
+      - ../../../monitoring-mixins/tempo-mixin/deploy/dashboards_out:/var/lib/grafana/dashboards/tempo-mixin
 
diff --git a/docker-compose/monolithic-mode/traces/mimirtool.override.yaml b/docker-compose/monolithic-mode/traces/mimirtool.override.yaml
@@ -5,5 +5,5 @@ services:
     volumes:
       - ../../../monitoring-mixins/loki-mixin/deploy/loki-mixin-rules.yaml:/rules/loki-mixin-rules.yaml
       - ../../../monitoring-mixins/loki-mixin/deploy/loki-mixin-alerts.yaml:/rules/loki-mixin-alerts.yaml
-      # - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/rules/tempo-mixin-rules.yaml
-      # - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/rules/tempo-mixin-alerts.yaml
+      - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-rules.yaml:/rules/tempo-mixin-rules.yaml
+      - ../../../monitoring-mixins/tempo-mixin/deploy/tempo-mixin-alerts.yaml:/rules/tempo-mixin-alerts.yaml
diff --git a/monitoring-mixins/agent-flow-mixin/deploy/agent-flow-mixin-alerts.yaml b/monitoring-mixins/agent-flow-mixin/deploy/agent-flow-mixin-alerts.yaml
@@ -3,20 +3,20 @@ groups:
       rules:
         - alert: ClusterNotConverging
           annotations:
-            message: Cluster is not converging.
+            message: 'Cluster is not converging: nodes report different number of peers in the cluster.'
           expr: stddev by (cluster, namespace) (sum without (state) (cluster_node_peers)) != 0
           for: 10m
-        - alert: ClusterSplitBrain
+        - alert: ClusterNodeCountMismatch
           annotations:
-            message: Cluster nodes have entered a split brain state.
+            message: Nodes report different number of peers vs. the count of observed agent metrics. Some agent metrics may be missing or the cluster is in a split brain state.
           expr: |
             sum without (state) (cluster_node_peers) !=
             on (cluster, namespace) group_left
             count by (cluster, namespace) (cluster_node_info)
-          for: 10m
+          for: 15m
         - alert: ClusterNodeUnhealthy
           annotations:
-            message: Cluster node is reporting a health score > 0.
+            message: Cluster node is reporting a gossip protocol health score > 0.
           expr: |
             cluster_node_gossip_health_score > 0
           for: 10m

diff --git a/monitoring-mixins/agent-flow-mixin/deploy/dashboards_out/agent-flow-opentelemetry.json b/monitoring-mixins/agent-flow-mixin/deploy/dashboards_out/agent-flow-opentelemetry.json
@@ -29,6 +29,17 @@
          {
             "datasource": "${datasource}",
             "description": "Number of spans successfully pushed into the pipeline.\n",
+            "fieldConfig": {
+               "defaults": {
+                  "custom": {
+                     "fillOpacity": 20,
+                     "gradientMode": "hue",
+                     "stacking": {
+                        "mode": "normal"
+                     }
+                  }
+               }
+            },
             "gridPos": {
                "h": 10,
                "w": 8,
@@ -82,11 +93,6 @@
          {
             "datasource": "${datasource}",
             "description": "The duration of inbound RPCs.\n",
-            "fieldConfig": {
-               "defaults": {
-                  "unit": "milliseconds"
-               }
-            },
             "gridPos": {
                "h": 10,
                "w": 8,
@@ -115,7 +121,7 @@
                   "yHistogram": true
                },
                "yAxis": {
-                  "unit": "s"
+                  "unit": "ms"
                }
             },
             "pluginVersion": "9.0.6",
@@ -129,7 +135,7 @@
                   "range": true
                }
             ],
-            "title": "RPC server duration (traces)",
+            "title": "RPC server duration",
             "type": "heatmap"
          },
          {
@@ -140,12 +146,17 @@
                "x": 0,
                "y": 10
             },
-            "title": "Batching [otelcol.processor.batch]",
+            "title": "Batching of logs, metrics, and traces [otelcol.processor.batch]",
             "type": "row"
          },
          {
             "datasource": "${datasource}",
-            "description": "Number of units in the batch\n",
+            "description": "Number of spans, metric datapoints, or log lines in a batch\n",
+            "fieldConfig": {
+               "defaults": {
+                  "unit": "short"
+               }
+            },
             "gridPos": {
                "h": 10,
                "w": 8,
@@ -174,7 +185,7 @@
                   "yHistogram": true
                },
                "yAxis": {
-                  "unit": "s"
+                  "unit": "short"
                }
             },
             "pluginVersion": "9.0.6",
@@ -247,6 +258,17 @@
          {
             "datasource": "${datasource}",
             "description": "Number of spans successfully sent to destination.\n",
+            "fieldConfig": {
+               "defaults": {
+                  "custom": {
+                     "fillOpacity": 20,
+                     "gradientMode": "hue",
+                     "stacking": {
+                        "mode": "normal"
+                     }
+                  }
+               }
+            },
             "gridPos": {
                "h": 10,
                "w": 8,
@@ -268,6 +290,17 @@
          {
             "datasource": "${datasource}",
             "description": "Number of spans in failed attempts to send to destination.\n",
+            "fieldConfig": {
+               "defaults": {
+                  "custom": {
+                     "fillOpacity": 20,
+                     "gradientMode": "hue",
+                     "stacking": {
+                        "mode": "normal"
+                     }
+                  }
+               }
+            },
             "gridPos": {
                "h": 10,
                "w": 8,

diff --git a/monitoring-mixins/agent-flow-mixin/deploy/manifests/k8s-all-in-one.yaml b/monitoring-mixins/agent-flow-mixin/deploy/manifests/k8s-all-in-one.yaml
@@ -1438,6 +1438,17 @@ data:
              {
                 "datasource": "${datasource}",
                 "description": "Number of spans successfully pushed into the pipeline.\n",
+                "fieldConfig": {
+                   "defaults": {
+                      "custom": {
+                         "fillOpacity": 20,
+                         "gradientMode": "hue",
+                         "stacking": {
+                            "mode": "normal"
+                         }
+                      }
+                   }
+                },
                 "gridPos": {
                    "h": 10,
                    "w": 8,
@@ -1491,11 +1502,6 @@ data:
              {
                 "datasource": "${datasource}",
                 "description": "The duration of inbound RPCs.\n",
-                "fieldConfig": {
-                   "defaults": {
-                      "unit": "milliseconds"
-                   }
-                },
                 "gridPos": {
                    "h": 10,
                    "w": 8,
@@ -1524,7 +1530,7 @@ data:
                       "yHistogram": true
                    },
                    "yAxis": {
-                      "unit": "s"
+                      "unit": "ms"
                    }
                 },
                 "pluginVersion": "9.0.6",
@@ -1538,7 +1544,7 @@ data:
                       "range": true
                    }
                 ],
-                "title": "RPC server duration (traces)",
+                "title": "RPC server duration",
                 "type": "heatmap"
              },
              {
@@ -1549,12 +1555,17 @@ data:
                    "x": 0,
                    "y": 10
                 },
-                "title": "Batching [otelcol.processor.batch]",
+                "title": "Batching of logs, metrics, and traces [otelcol.processor.batch]",
                 "type": "row"
              },
              {
                 "datasource": "${datasource}",
-                "description": "Number of units in the batch\n",
+                "description": "Number of spans, metric datapoints, or log lines in a batch\n",
+                "fieldConfig": {
+                   "defaults": {
+                      "unit": "short"
+                   }
+                },
                 "gridPos": {
                    "h": 10,
                    "w": 8,
@@ -1583,7 +1594,7 @@ data:
                       "yHistogram": true
                    },
                    "yAxis": {
-                      "unit": "s"
+                      "unit": "short"
                    }
                 },
                 "pluginVersion": "9.0.6",
@@ -1656,6 +1667,17 @@ data:
              {
                 "datasource": "${datasource}",
                 "description": "Number of spans successfully sent to destination.\n",
+                "fieldConfig": {
+                   "defaults": {
+                      "custom": {
+                         "fillOpacity": 20,
+                         "gradientMode": "hue",
+                         "stacking": {
+                            "mode": "normal"
+                         }
+                      }
+                   }
+                },
                 "gridPos": {
                    "h": 10,
                    "w": 8,
@@ -1677,6 +1699,17 @@ data:
              {
                 "datasource": "${datasource}",
                 "description": "Number of spans in failed attempts to send to destination.\n",
+                "fieldConfig": {
+                   "defaults": {
+                      "custom": {
+                         "fillOpacity": 20,
+                         "gradientMode": "hue",
+                         "stacking": {
+                            "mode": "normal"
+                         }
+                      }
+                   }
+                },
                 "gridPos": {
                    "h": 10,
                    "w": 8,
@@ -2807,37 +2840,27 @@ spec:
     rules:
     - alert: ClusterNotConverging
       annotations:
-        message: Cluster is not converging.
+        message: 'Cluster is not converging: nodes report different number of peers
+          in the cluster.'
       expr: stddev by (cluster, namespace) (sum without (state) (cluster_node_peers))
         != 0
-      for: 5m
-    - alert: ClusterSplitBrain
+      for: 10m
+    - alert: ClusterNodeCountMismatch
       annotations:
-        message: Cluster nodes have entered a split brain state.
+        message: Nodes report different number of peers vs. the count of observed
+          agent metrics. Some agent metrics may be missing or the cluster is in a
+          split brain state.
       expr: |
         sum without (state) (cluster_node_peers) !=
         on (cluster, namespace) group_left
         count by (cluster, namespace) (cluster_node_info)
-      for: 5m
-    - alert: ClusterLamportClockDrift
-      annotations:
-        message: Cluster nodes' lamport clocks are not converging.
-      expr: stddev by (cluster, namespace) (cluster_node_lamport_time) > 4 * sqrt(count
-        by (cluster, namespace) (cluster_node_info))
-      for: 5m
+      for: 15m
     - alert: ClusterNodeUnhealthy
       annotations:
-        message: Cluster node is reporting a health score > 0.
+        message: Cluster node is reporting a gossip protocol health score > 0.
       expr: |
         cluster_node_gossip_health_score > 0
-      for: 5m
-    - alert: ClusterLamportClockStuck
-      annotations:
-        message: Cluster nodes's lamport clocks is not progressing.
-      expr: |
-        sum by (cluster, namespace, instance) (rate(cluster_node_lamport_time[2m])) == 0
-        and on (cluster, namespace, instance) (cluster_node_peers > 1)
-      for: 5m
+      for: 10m
     - alert: ClusterNodeNameConflict
       annotations:
         message: A node tried to join the cluster with a name conflicting with an
@@ -2850,7 +2873,7 @@ spec:
         message: Cluster node stuck in Terminating state.
       expr: sum by (cluster, namespace, instance) (cluster_node_peers{state="terminating"})
         > 0
-      for: 5m
+      for: 10m
     - alert: ClusterConfigurationDrift
       annotations:
         message: Cluster nodes are not using the same configuration file.
@@ -2870,6 +2893,6 @@ spec:
     - alert: UnhealthyComponents
       annotations:
         message: Unhealthy Flow components detected.
-      expr: sum(agent_component_controller_running_components{health_type!="healthy"})
+      expr: sum by (cluster, namespace) (agent_component_controller_running_components{health_type!="healthy"})
         > 0
       for: 15m