[raft] add some metrics to monitor zombies (#8184)

buildbuddy-io · Jan 15, 2025 · 4987ddd · 4987ddd
1 parent a6ba7df
commit 4987ddd
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 0 deletions.
diff --git a/enterprise/server/raft/store/store.go b/enterprise/server/raft/store/store.go
@@ -1534,8 +1534,10 @@ func (j *replicaJanitor) Start(ctx context.Context) {
 			case task := <-j.tasks:
 				action, err := j.removeZombie(ctx, task)
 				if err != nil {
+					j.store.log.Errorf("failed to remove zombie c%dn%d: %s", task.rangeID, task.replicaID, err)
 					task.action = action
 					j.tasks <- task
+					metrics.RaftZombieCleanupErrorCount.Inc()
 				} else {
 					j.mu.Lock()
 					delete(j.rangeIDsInQueue, task.rangeID)
@@ -1616,6 +1618,7 @@ func (j *replicaJanitor) scan(ctx context.Context) {
 					}
 				}
 			}
+			metrics.RaftZombieCleanupTasks.Set(float64(len(j.tasks)))
 		}
 	}
 }

diff --git a/server/metrics/metrics.go b/server/metrics/metrics.go
@@ -2345,6 +2345,20 @@ var (
 		StatusHumanReadableLabel,
 	})
 
+	RaftZombieCleanupTasks = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: bbNamespace,
+		Subsystem: "raft",
+		Name:      "zombie_cleanup_tasks",
+		Help:      "The total number of pending zombie cleanup tasks",
+	})
+
+	RaftZombieCleanupErrorCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: bbNamespace,
+		Subsystem: "raft",
+		Name:      "zombie_cleanup_errors",
+		Help:      "The total number of zombie cleanup errors",
+	})
+
 	APIKeyLookupCount = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: bbNamespace,
 		Subsystem: "auth",