add isTerminating prometheus metric (#622)

* add isTerminating prometheus metric * terminating -> disabled, lower shutdown check timer
livekit · Feb 29, 2024 · bdbbd99 · bdbbd99
1 parent 6bf97e0
commit bdbbd99
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 5 deletions.
diff --git a/pkg/service/service.go b/pkg/service/service.go
@@ -39,8 +39,6 @@ import (
 	"github.com/livekit/protocol/rpc"
 )
 
-const shutdownTimer = time.Second * 30
-
 type Service struct {
 	ipc.UnimplementedEgressServiceServer
 
@@ -89,6 +87,7 @@ func NewService(conf *config.ServiceConfig, ioClient rpc.IOInfoClient) (*Service
 	if err := s.Start(s.conf,
 		s.promIsIdle,
 		s.promCanAcceptRequest,
+		s.promIsDisabled,
 		s.killProcess,
 	); err != nil {
 		return nil, err
@@ -206,7 +205,7 @@ func (s *Service) killProcess(egressID string, maxUsage float64) {
 
 func (s *Service) Close() {
 	for s.GetRequestCount() > 0 {
-		time.Sleep(shutdownTimer)
+		time.Sleep(time.Second)
 	}
 	logger.Infow("closing server")
 	s.psrpcServer.Shutdown()

diff --git a/pkg/service/service_prom.go b/pkg/service/service_prom.go
@@ -69,7 +69,7 @@ func (s *Service) PromHandler() http.Handler {
 }
 
 func (s *Service) promIsIdle() float64 {
-	if s.GetRequestCount() == 0 {
+	if !s.shutdown.IsBroken() && s.GetRequestCount() == 0 {
 		return 1
 	}
 	return 0
@@ -89,6 +89,13 @@ func (s *Service) promCanAcceptRequest() float64 {
 	return 0
 }
 
+func (s *Service) promIsDisabled() float64 {
+	if s.shutdown.IsBroken() {
+		return 1
+	}
+	return 0
+}
+
 func (s *Service) storeProcessEndedMetrics(egressID string, metrics string) error {
 	m, err := deserializeMetrics(egressID, metrics)
 	if err != nil {

diff --git a/pkg/stats/monitor.go b/pkg/stats/monitor.go
@@ -77,6 +77,7 @@ func (m *Monitor) Start(
 	conf *config.ServiceConfig,
 	isIdle func() float64,
 	canAcceptRequest func() float64,
+	isDisabled func() float64,
 	killProcess func(string, float64),
 ) error {
 	m.killProcess = killProcess
@@ -105,6 +106,13 @@ func (m *Monitor) Start(
 		ConstLabels: prometheus.Labels{"node_id": conf.NodeID, "cluster_id": conf.ClusterID},
 	}, canAcceptRequest)
 
+	promIsDisabled := prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace:   "livekit",
+		Subsystem:   "egress",
+		Name:        "is_disabled",
+		ConstLabels: prometheus.Labels{"node_id": conf.NodeID, "cluster_id": conf.ClusterID},
+	}, isDisabled)
+
 	m.promCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{
 		Namespace:   "livekit",
 		Subsystem:   "node",
@@ -119,7 +127,7 @@ func (m *Monitor) Start(
 		ConstLabels: prometheus.Labels{"node_id": conf.NodeID, "cluster_id": conf.ClusterID},
 	}, []string{"type"})
 
-	prometheus.MustRegister(promNodeAvailable, promCanAcceptRequest, m.promCPULoad, m.requestGauge)
+	prometheus.MustRegister(promNodeAvailable, promCanAcceptRequest, promIsDisabled, m.promCPULoad, m.requestGauge)
 
 	return nil
 }