From a6f0c8aa515567b05b5da3820b3aa879d33a1ce4 Mon Sep 17 00:00:00 2001 From: SuperQ Date: Tue, 8 Feb 2022 19:28:09 +0100 Subject: [PATCH] Add Prometheus metrics to redisbp Add a Prometheus exporter interface implementation to the redisbp `NewMonitoredClient()`. Signed-off-by: SuperQ --- redis/db/redisbp/monitored_client.go | 21 +++++ redis/db/redisbp/prometheus.go | 128 +++++++++++++++++++++++++++ redis/db/redisbp/prometheus_test.go | 46 ++++++++++ 3 files changed, 195 insertions(+) create mode 100644 redis/db/redisbp/prometheus.go create mode 100644 redis/db/redisbp/prometheus_test.go diff --git a/redis/db/redisbp/monitored_client.go b/redis/db/redisbp/monitored_client.go index 5291a7dc0..33eed54f3 100644 --- a/redis/db/redisbp/monitored_client.go +++ b/redis/db/redisbp/monitored_client.go @@ -7,6 +7,7 @@ import ( "time" "github.com/go-redis/redis/v8" + "github.com/prometheus/client_golang/prometheus" "github.com/reddit/baseplate.go/metricsbp" ) @@ -25,6 +26,13 @@ type PoolStatser interface { func NewMonitoredClient(name string, opt *redis.Options) *redis.Client { client := redis.NewClient(opt) client.AddHook(SpanHook{ClientName: name}) + + if err := prometheus.Register(newExporter(client, name)); err != nil { + // prometheus.Register should never fail because + // exporter.Describe is a no-op, but just in case. + return nil + } + return client } @@ -33,6 +41,13 @@ func NewMonitoredClient(name string, opt *redis.Options) *redis.Client { func NewMonitoredFailoverClient(name string, opt *redis.FailoverOptions) *redis.Client { client := redis.NewFailoverClient(opt) client.AddHook(SpanHook{ClientName: name}) + + if err := prometheus.Register(newExporter(client, name)); err != nil { + // prometheus.Register should never fail because + // exporter.Describe is a no-op, but just in case. + return nil + } + return client } @@ -78,6 +93,12 @@ func NewMonitoredClusterClient(name string, opt *redis.ClusterOptions) *ClusterC client := redis.NewClusterClient(opt) client.AddHook(SpanHook{ClientName: name}) + if err := prometheus.Register(newExporter(client, name)); err != nil { + // prometheus.Register should never fail because + // exporter.Describe is a no-op, but just in case. + return nil + } + return &ClusterClient{client} } diff --git a/redis/db/redisbp/prometheus.go b/redis/db/redisbp/prometheus.go new file mode 100644 index 000000000..98ac43423 --- /dev/null +++ b/redis/db/redisbp/prometheus.go @@ -0,0 +1,128 @@ +package redisbp + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const ( + promNamespace = "redisbp" + subsystemPool = "pool" + + nameLabel = "pool" +) + +// exporter provides an interface for Prometheus metrics. +type exporter struct { + client PoolStatser + name string + + poolHitsCounterDesc *prometheus.Desc + poolMissesCounterDesc *prometheus.Desc + poolTimeoutsCounterDesc *prometheus.Desc + totalConnectionsDesc *prometheus.Desc + idleConnectionsDesc *prometheus.Desc + staleConnectionsDesc *prometheus.Desc +} + +func newExporter(client PoolStatser, name string) *exporter { + labels := []string{ + nameLabel, + } + + return &exporter{ + client: client, + name: name, + + // Upstream docs: https://pkg.go.dev/github.com/go-redis/redis/v8/internal/pool#Stats + + // Counters. + poolHitsCounterDesc: prometheus.NewDesc( + prometheus.BuildFQName(promNamespace, subsystemPool, "hits_total"), + "Number of times free connection was found in the pool", + labels, + nil, + ), + poolMissesCounterDesc: prometheus.NewDesc( + prometheus.BuildFQName(promNamespace, subsystemPool, "misses_total"), + "Number of times free connection was NOT found in the pool", + labels, + nil, + ), + poolTimeoutsCounterDesc: prometheus.NewDesc( + prometheus.BuildFQName(promNamespace, subsystemPool, "timeouts_total"), + "Number of times a wait timeout occurred", + labels, + nil, + ), + + // Gauges. + totalConnectionsDesc: prometheus.NewDesc( + prometheus.BuildFQName(promNamespace, subsystemPool, "connections"), + "Number of connections in this redisbp pool", + labels, + nil, + ), + idleConnectionsDesc: prometheus.NewDesc( + prometheus.BuildFQName(promNamespace, subsystemPool, "idle_connections"), + "Number of idle connections in this redisbp pool", + labels, + nil, + ), + staleConnectionsDesc: prometheus.NewDesc( + prometheus.BuildFQName(promNamespace, subsystemPool, "stale_connections"), + "Number of stale connections in this redisbp pool", + labels, + nil, + ), + } +} + +// Describe implements the prometheus.Collector interface. +func (e *exporter) Describe(ch chan<- *prometheus.Desc) { + // All metrics are described dynamically. +} + +// Collect implements prometheus.Collector. +func (e *exporter) Collect(ch chan<- prometheus.Metric) { + stats := e.client.PoolStats() + + // Counters. + ch <- prometheus.MustNewConstMetric( + e.poolHitsCounterDesc, + prometheus.CounterValue, + float64(stats.Hits), + e.name, + ) + ch <- prometheus.MustNewConstMetric( + e.poolMissesCounterDesc, + prometheus.CounterValue, + float64(stats.Misses), + e.name, + ) + ch <- prometheus.MustNewConstMetric( + e.poolTimeoutsCounterDesc, + prometheus.CounterValue, + float64(stats.Timeouts), + e.name, + ) + + // Gauges. + ch <- prometheus.MustNewConstMetric( + e.totalConnectionsDesc, + prometheus.GaugeValue, + float64(stats.TotalConns), + e.name, + ) + ch <- prometheus.MustNewConstMetric( + e.idleConnectionsDesc, + prometheus.GaugeValue, + float64(stats.IdleConns), + e.name, + ) + ch <- prometheus.MustNewConstMetric( + e.staleConnectionsDesc, + prometheus.GaugeValue, + float64(stats.StaleConns), + e.name, + ) +} diff --git a/redis/db/redisbp/prometheus_test.go b/redis/db/redisbp/prometheus_test.go new file mode 100644 index 000000000..6caf5dcc8 --- /dev/null +++ b/redis/db/redisbp/prometheus_test.go @@ -0,0 +1,46 @@ +package redisbp + +import ( + "sync" + "testing" + + "github.com/go-redis/redis/v8" + "github.com/prometheus/client_golang/prometheus" +) + +type fakeClient redis.Client + +// PoolStats returns connection pool stats. +func (c fakeClient) PoolStats() *redis.PoolStats { + return &redis.PoolStats{ + Hits: 1, + Misses: 2, + Timeouts: 3, + + TotalConns: 4, + IdleConns: 5, + StaleConns: 6, + } +} + +func TestRedisPoolExporter(t *testing.T) { + client := &fakeClient{} + + exporter := newExporter( + client, + "test", + ) + // No real test here, we just want to make sure that Collect call will not + // panic, which would happen if we have a label mismatch. + ch := make(chan prometheus.Metric) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for range ch { + } + }() + exporter.Collect(ch) + close(ch) + wg.Wait() +}