From 38d9f05de54e8e561f031c8ca77f4c94799424b1 Mon Sep 17 00:00:00 2001 From: Alex Snaps Date: Thu, 21 Dec 2023 09:50:29 -0500 Subject: [PATCH] Added datastore latency histogram to prometheus metrics --- Cargo.lock | 2 ++ limitador/Cargo.toml | 2 ++ limitador/src/lib.rs | 8 +++++++ limitador/src/prometheus_metrics.rs | 34 ++++++++++++++++++++++++++++- 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index efb41d25..8359b76e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1516,6 +1516,7 @@ dependencies = [ "getrandom", "infinispan", "lazy_static", + "log", "moka", "paste", "postcard", @@ -1533,6 +1534,7 @@ dependencies = [ "thiserror", "tokio", "tracing", + "tracing-subscriber", "ttl_cache", ] diff --git a/limitador/Cargo.toml b/limitador/Cargo.toml index 0f2336b7..41f54798 100644 --- a/limitador/Cargo.toml +++ b/limitador/Cargo.toml @@ -35,6 +35,7 @@ cfg-if = "1" prometheus = "0.13" lazy_static = "1" tracing = "0.1.40" +tracing-subscriber = "0.3.18" # Optional dependencies rocksdb = { version = "0.21.0", optional = true, features = ["multi-threaded-cf"] } @@ -53,6 +54,7 @@ tokio = { version = "1", optional = true, features = [ infinispan = { version = "0.3", optional = true } reqwest = { version = "0.11", optional = true } base64 = { version = "0.21.0", optional = true } +log = "0.4.20" [dev-dependencies] serial_test = "2.0.0" diff --git a/limitador/src/lib.rs b/limitador/src/lib.rs index 177ae1d0..838ab035 100644 --- a/limitador/src/lib.rs +++ b/limitador/src/lib.rs @@ -323,6 +323,10 @@ impl RateLimiter { } } + pub fn metrics(&self) -> &PrometheusMetrics { + &self.prometheus_metrics + } + pub fn get_namespaces(&self) -> HashSet { self.storage.get_namespaces() } @@ -508,6 +512,10 @@ impl AsyncRateLimiter { } } + pub fn metrics(&self) -> &PrometheusMetrics { + &self.prometheus_metrics + } + pub fn get_namespaces(&self) -> HashSet { self.storage.get_namespaces() } diff --git a/limitador/src/prometheus_metrics.rs b/limitador/src/prometheus_metrics.rs index 9704a7f3..b7cc09f4 100644 --- a/limitador/src/prometheus_metrics.rs +++ b/limitador/src/prometheus_metrics.rs @@ -1,5 +1,8 @@ use crate::limit::Namespace; -use prometheus::{Encoder, IntCounterVec, IntGauge, Opts, Registry, TextEncoder}; +use prometheus::{ + Encoder, Histogram, HistogramOpts, IntCounterVec, IntGauge, Opts, Registry, TextEncoder, +}; +use std::time::Duration; const NAMESPACE_LABEL: &str = "limitador_namespace"; const LIMIT_NAME_LABEL: &str = "limit_name"; @@ -22,15 +25,26 @@ lazy_static! { name: "limitador_up".into(), description: "Limitador is running".into(), }; + static ref DATASTORE_LATENCY: Metric = Metric { + name: "counter_latency".into(), + description: "Latency to the underlying counter datastore".into(), + }; } pub struct PrometheusMetrics { registry: Registry, authorized_calls: IntCounterVec, limited_calls: IntCounterVec, + counter_latency: Histogram, use_limit_name_label: bool, } +impl Default for PrometheusMetrics { + fn default() -> Self { + Self::new() + } +} + impl PrometheusMetrics { pub fn new() -> Self { Self::new_with_options(false) @@ -65,6 +79,10 @@ impl PrometheusMetrics { self.limited_calls.with_label_values(&labels).inc(); } + pub fn counter_access(&self, duration: Duration) { + self.counter_latency.observe(duration.as_secs_f64()); + } + pub fn gather_metrics(&self) -> String { let mut buffer = Vec::new(); @@ -79,6 +97,7 @@ impl PrometheusMetrics { let authorized_calls_counter = Self::authorized_calls_counter(); let limited_calls_counter = Self::limited_calls_counter(use_limit_name_label); let limitador_up_gauge = Self::limitador_up_gauge(); + let counter_latency = Self::counter_latency(); let registry = Registry::new(); @@ -94,12 +113,17 @@ impl PrometheusMetrics { .register(Box::new(limitador_up_gauge.clone())) .unwrap(); + registry + .register(Box::new(counter_latency.clone())) + .unwrap(); + limitador_up_gauge.set(1); Self { registry, authorized_calls: authorized_calls_counter, limited_calls: limited_calls_counter, + counter_latency, use_limit_name_label, } } @@ -129,6 +153,14 @@ impl PrometheusMetrics { fn limitador_up_gauge() -> IntGauge { IntGauge::new(&LIMITADOR_UP.name, &LIMITADOR_UP.description).unwrap() } + + fn counter_latency() -> Histogram { + Histogram::with_opts(HistogramOpts::new( + &DATASTORE_LATENCY.name, + &DATASTORE_LATENCY.description, + )) + .unwrap() + } } #[cfg(test)]