クラスターメトリクスに切断中のノードを含め、接続状態を gauge の値で返す

CHANGES.md

@@ -1,5 +1,15 @@
 # CHANGES
 
+## develop
+
+- [FIX] Sora 2023.2.0 で `ListClusterNodes` API の `include_all_known_nodes` のデフォルト値が変更で panic が起こす問題に対応する
+  - Sora 2023.2.0 以降で Sora Exporter 2023.5.0 以前のバージョンを使用し、クラスターメトリクスが有効になっている場合に発生する
+  - @tnamao
+- [CHANGE] Sora の `ListClusterNodes` API を呼び出す際に、API リクエストの `include_all_known_nodes` を `true` にし切断中のノードも含め、接続状態を gauge で返すようにする
+  - **破壊的変更** になるため、バージョンアップの際に注意してください
+  - gauge の値は 1 が接続、0 が切断を表し `ListClusterNodes` API のレスポンスに含まれる `connected` の値により返す値を切り替えている
+  - @tnamao
+
 ## 2023.5.0
 
 - [UPDATE] CI の staticcheck のバージョンを 2023.1.6 に上げる

collector/cluster_node.go

@@ -1,6 +1,8 @@
 package collector
 
-import "github.com/prometheus/client_golang/prometheus"
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
 
 var (
 	soraClusterMetrics = SoraClusterMetrics{
@@ -27,10 +29,14 @@ func (m *SoraClusterMetrics) Describe(ch chan<- *prometheus.Desc) {
 
 func (m *SoraClusterMetrics) Collect(ch chan<- prometheus.Metric, nodeList []soraClusterNode, report soraClusterReport) {
 	for _, node := range nodeList {
-		if node.ClusterNodeName != nil {
-			ch <- newGauge(m.clusterNode, 1, *node.ClusterNodeName, *node.Mode)
+		value := 0.0
+		if node.Connected {
+			value = 1.0
+		}
+		if node.ClusterNodeName != "" {
+			ch <- newGauge(m.clusterNode, value, node.ClusterNodeName, node.Mode)
 		} else {
-			ch <- newGauge(m.clusterNode, 1, *node.NodeName, *node.Mode)
+			ch <- newGauge(m.clusterNode, value, node.NodeName, node.Mode)
 		}
 	}
 	ch <- newGauge(m.raftState, 1.0, report.RaftState)

collector/collector.go

@@ -1,6 +1,7 @@
 package collector
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"encoding/json"
@@ -50,6 +51,10 @@ type HTTPClient interface {
 	Do(*http.Request) (*http.Response, error)
 }
 
+type SoraListClusterNodesRequest struct {
+	IncludeAllKnownNodes bool `json:"include_all_known_nodes"`
+}
+
 func NewCollector(options *CollectorOptions) *Collector {
 	return &Collector{
 		URI:           options.URI,
@@ -112,7 +117,17 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
 
 	var nodeList []soraClusterNode
 	if c.EnableSoraClusterMetrics {
-		req, err = http.NewRequestWithContext(ctx, http.MethodPost, c.URI, nil)
+		requestParams := SoraListClusterNodesRequest{
+			IncludeAllKnownNodes: true,
+		}
+		encodedParams, err := json.Marshal(requestParams)
+		if err != nil {
+			level.Error(c.logger).Log("msg", "failed to encode Sora ListClusterNodes API request parameters", "err", err)
+			ch <- newGauge(c.soraUp, 0)
+			return
+		}
+
+		req, err = http.NewRequestWithContext(ctx, http.MethodPost, c.URI, bytes.NewBuffer(encodedParams))
 		if err != nil {
 			level.Error(c.logger).Log("msg", "failed to create request to sora", "err", err)
 			ch <- newGauge(c.soraUp, 0)

collector/sora_api.go

@@ -136,9 +136,10 @@ type soraClusterReport struct {
 }
 
 type soraClusterNode struct {
-	ClusterNodeName *string `json:"cluster_node_name"`
-	NodeName        *string `json:"node_name"`
-	Mode            *string `json:"mode"`
+	ClusterNodeName string `json:"cluster_node_name"`
+	NodeName        string `json:"node_name"`
+	Mode            string `json:"mode"`
+	Connected       bool   `json:"connected"`
 }
 
 type soraLicenseInfo struct {

main_test.go

@@ -150,17 +150,7 @@ var (
 	listClusterNodesJSONData = `[
 		{
 		  "node_name": "[email protected]",
-		  "epoch": 1,
-		  "mode": "normal",
-		  "cluster_signaling_url": "ws://127.0.0.1:5001/signaling",
-		  "cluster_api_url": "http://127.0.0.1:3101/",
-		  "member_since": "2022-05-09T07:44:52.973761Z",
-		  "sora_version": "2022.1.0-canary.44",
-		  "license_max_nodes": 10,
-		  "license_max_connections": 100,
-		  "license_serial_code": "SAMPLE-SRA-E001-202212-N10-100",
-		  "license_type": "Experimental",
-		  "connected": true
+		  "connected": false
 		},
 		{
 		  "node_name": "[email protected]",
@@ -175,20 +165,25 @@ var (
 		  "license_serial_code": "SAMPLE-SRA-E001-202212-N10-100",
 		  "license_type": "Experimental",
 		  "connected": true
-		}
-	  ]`
+		},
+		{
+			"node_name": "[email protected]",
+			"epoch": 1,
+			"mode": "normal",
+			"cluster_signaling_url": "ws://127.0.0.1:5001/signaling",
+			"cluster_api_url": "http://127.0.0.1:3101/",
+			"member_since": "2022-05-09T07:44:54.160763Z",
+			"sora_version": "2022.1.0-canary.44",
+			"license_max_nodes": 10,
+			"license_max_connections": 100,
+			"license_serial_code": "SAMPLE-SRA-E001-202212-N10-100",
+			"license_type": "Experimental",
+			"connected": true
+		  }
+		]`
 	listClusterNodesCurrentJSONData = `[
 		{
 		  "cluster_node_name": "[email protected]",
-		  "epoch": 1,
-		  "mode": "normal",
-		  "member_since": "2022-05-02T15:26:44.302363Z",
-		  "sora_version": "2021.2.9",
-		  "license_max_connections": 100,
-		  "license_serial_code": "SAMPLE-SRA-E001-202212-N10-100",
-		  "license_type": "Experimental",
-		  "cluster_signaling_url": "ws://127.0.0.1:5001/signaling",
-		  "cluster_api_url": "http://10.1.1.4:3000/",
 		  "connected": false
 		},
 		{
@@ -203,7 +198,21 @@ var (
 		  "cluster_signaling_url": "ws://127.0.0.1:5002/signaling",
 		  "cluster_api_url": "http://10.1.1.3:3000/",
 		  "connected": true
-		}
+		},
+		{
+			"node_name": "[email protected]",
+			"epoch": 1,
+			"mode": "normal",
+			"cluster_signaling_url": "ws://127.0.0.1:5001/signaling",
+			"cluster_api_url": "http://127.0.0.1:3101/",
+			"member_since": "2022-05-09T07:44:54.160763Z",
+			"sora_version": "2022.1.0-canary.44",
+			"license_max_nodes": 10,
+			"license_max_connections": 100,
+			"license_serial_code": "SAMPLE-SRA-E001-202212-N10-100",
+			"license_type": "Experimental",
+			"connected": true
+		  }
 	  ]`
 	getLicenseJSONDATA = `{
 		"expired_at": "2025-09",

test/maximum.metrics

@@ -10,8 +10,9 @@ sora_average_duration_seconds 706
 sora_average_setup_time_seconds 0
 # HELP sora_cluster_node The sora server known cluster node.
 # TYPE sora_cluster_node gauge
+sora_cluster_node{mode="",node_name="[email protected]"} 0
 sora_cluster_node{mode="block_new_connection",node_name="[email protected]"} 1
-sora_cluster_node{mode="normal",node_name="node-01_canary_sora@10.211.55.42"} 1
+sora_cluster_node{mode="normal",node_name="node-03_canary_sora@10.211.55.41"} 1
 # HELP sora_cluster_raft_commit_index The latest committed Raft log index.
 # TYPE sora_cluster_raft_commit_index counter
 sora_cluster_raft_commit_index 10

test/sora_cluster_metrics_enabled.metrics

@@ -10,8 +10,9 @@ sora_average_duration_seconds 706
 sora_average_setup_time_seconds 0
 # HELP sora_cluster_node The sora server known cluster node.
 # TYPE sora_cluster_node gauge
+sora_cluster_node{mode="normal",node_name="[email protected]"} 1
 sora_cluster_node{mode="block_new_connection",node_name="[email protected]"} 1
-sora_cluster_node{mode="normal",node_name="[email protected]"} 1
+sora_cluster_node{mode="",node_name="[email protected]"} 0
 # HELP sora_cluster_raft_commit_index The latest committed Raft log index.
 # TYPE sora_cluster_raft_commit_index counter
 sora_cluster_raft_commit_index 10