From 9321f60a2de69ce433086c907d24203ef165edec Mon Sep 17 00:00:00 2001 From: Croxx Date: Thu, 28 Nov 2024 15:16:56 +0800 Subject: [PATCH] feat: support prometheus-client metrics exporter (#805) * feat: support prometheus-client metrics exporter Signed-off-by: MrCroxx * test: text encode in ut Signed-off-by: MrCroxx --------- Signed-off-by: MrCroxx --- .github/workflows/ci.yml | 4 +- Cargo.toml | 1 + Makefile | 6 +- foyer-common/Cargo.toml | 3 + foyer-common/src/metrics/mod.rs | 6 +- foyer-common/src/metrics/model.rs | 14 ++ foyer-common/src/metrics/registry/mod.rs | 8 + foyer-common/src/metrics/registry/noop.rs | 6 +- .../metrics/registry/opentelemetry_0_26.rs | 6 +- .../metrics/registry/opentelemetry_0_27.rs | 6 +- .../src/metrics/registry/prometheus.rs | 8 +- .../registry/prometheus_client_0_22.rs | 232 ++++++++++++++++++ foyer/Cargo.toml | 2 + foyer/src/prelude.rs | 4 + 14 files changed, 285 insertions(+), 21 deletions(-) create mode 100644 foyer-common/src/metrics/registry/prometheus_client_0_22.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bb0c12a..af66656a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -165,7 +165,7 @@ jobs: cargo clippy --all-targets --features tokio-console -- -D warnings cargo clippy --all-targets --features deadlock -- -D warnings cargo clippy --all-targets --features tracing -- -D warnings - cargo clippy --all-targets --features prometheus,opentelemetry_0_26,opentelemetry_0_27 -- -D warnings + cargo clippy --all-targets --features prometheus,prometheus-client_0_22,opentelemetry_0_26,opentelemetry_0_27 -- -D warnings cargo clippy --all-targets -- -D warnings - if: steps.cache.outputs.cache-hit != 'true' uses: taiki-e/install-action@cargo-llvm-cov @@ -182,7 +182,7 @@ jobs: RUST_BACKTRACE: 1 CI: true run: | - cargo llvm-cov --no-report nextest --features "strict_assertions,sanity,prometheus,opentelemetry_0_26,opentelemetry_0_27" + cargo llvm-cov --no-report nextest --features "strict_assertions,sanity,prometheus,prometheus-client_0_22,opentelemetry_0_26,opentelemetry_0_27" - name: Run examples with coverage env: RUST_BACKTRACE: 1 diff --git a/Cargo.toml b/Cargo.toml index e01b3d65..fe308b1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,7 @@ tracing = "0.1" prometheus = "0.13" opentelemetry_0_27 = { package = "opentelemetry", version = "0.27" } opentelemetry_0_26 = { package = "opentelemetry", version = "0.26" } +prometheus-client_0_22 = { package = "prometheus-client", version = "0.22" } # foyer components foyer-common = { version = "0.13.0-dev", path = "foyer-common" } diff --git a/Makefile b/Makefile index b836120b..7ea82934 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ check: ./scripts/minimize-dashboards.sh cargo sort -w cargo fmt --all - cargo clippy --all-targets --features prometheus,opentelemetry_0_26,opentelemetry_0_27 + cargo clippy --all-targets --features prometheus,prometheus-client_0_22,opentelemetry_0_26,opentelemetry_0_27 check-all: shellcheck ./scripts/* @@ -21,11 +21,11 @@ check-all: cargo clippy --all-targets --features tokio-console cargo clippy --all-targets --features sanity cargo clippy --all-targets --features tracing - cargo clippy --all-targets --features prometheus,opentelemetry_0_26,opentelemetry_0_27 + cargo clippy --all-targets --features prometheus,prometheus-client_0_22,opentelemetry_0_26,opentelemetry_0_27 cargo clippy --all-targets test: - RUST_BACKTRACE=1 cargo nextest run --all --features "strict_assertions,sanity,prometheus,opentelemetry_0_26,opentelemetry_0_27" + RUST_BACKTRACE=1 cargo nextest run --all --features "strict_assertions,sanity,prometheus,prometheus-client_0_22,opentelemetry_0_26,opentelemetry_0_27" RUST_BACKTRACE=1 cargo test --doc test-ignored: diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml index 10f9fcde..ab862bb8 100644 --- a/foyer-common/Cargo.toml +++ b/foyer-common/Cargo.toml @@ -25,6 +25,7 @@ opentelemetry_0_27 = { workspace = true, optional = true } parking_lot = { workspace = true } pin-project = "1" prometheus = { workspace = true, optional = true } +prometheus-client_0_22 = { workspace = true, optional = true } serde = { workspace = true } tokio = { workspace = true } @@ -36,6 +37,8 @@ rand = "0.8.5" strict_assertions = [] tracing = ["fastrace/enable"] prometheus = ["dep:prometheus"] +prometheus-client = ["prometheus-client_0_22"] +prometheus-client_0_22 = ["dep:prometheus-client_0_22"] opentelemetry = ["opentelemetry_0_27"] opentelemetry_0_27 = ["dep:opentelemetry_0_27"] opentelemetry_0_26 = ["dep:opentelemetry_0_26"] diff --git a/foyer-common/src/metrics/mod.rs b/foyer-common/src/metrics/mod.rs index 576182f7..8e63bb8f 100644 --- a/foyer-common/src/metrics/mod.rs +++ b/foyer-common/src/metrics/mod.rs @@ -39,19 +39,19 @@ pub trait HistogramOps: Send + Sync + 'static + Debug { /// A vector of counters. pub trait CounterVecOps: Send + Sync + 'static + Debug { /// Get a counter within the vector of counters. - fn counter(&self, labels: &[&str]) -> impl CounterOps; + fn counter(&self, labels: &[&'static str]) -> impl CounterOps; } /// A vector of gauges. pub trait GaugeVecOps: Send + Sync + 'static + Debug { /// Get a gauge within the vector of gauges. - fn gauge(&self, labels: &[&str]) -> impl GaugeOps; + fn gauge(&self, labels: &[&'static str]) -> impl GaugeOps; } /// A vector of histograms. pub trait HistogramVecOps: Send + Sync + 'static + Debug { /// Get a histogram within the vector of histograms. - fn histogram(&self, labels: &[&str]) -> impl HistogramOps; + fn histogram(&self, labels: &[&'static str]) -> impl HistogramOps; } /// Metrics registry. diff --git a/foyer-common/src/metrics/model.rs b/foyer-common/src/metrics/model.rs index 233f03a6..f8093cef 100644 --- a/foyer-common/src/metrics/model.rs +++ b/foyer-common/src/metrics/model.rs @@ -366,6 +366,20 @@ mod tests { case(&PrometheusMetricsRegistry::new(prometheus::Registry::new())); } + #[cfg(feature = "prometheus-client_0_22")] + #[test] + fn test_metrics_prometheus_client_0_22() { + use std::sync::Arc; + + use parking_lot::Mutex; + + use crate::metrics::registry::prometheus_client_0_22::PrometheusClientMetricsRegistry; + + case(&PrometheusClientMetricsRegistry::new(Arc::new(Mutex::new( + prometheus_client_0_22::registry::Registry::default(), + )))); + } + #[cfg(feature = "opentelemetry_0_27")] #[test] fn test_metrics_opentelemetry_0_27() { diff --git a/foyer-common/src/metrics/registry/mod.rs b/foyer-common/src/metrics/registry/mod.rs index a88fecc5..4a1194a2 100644 --- a/foyer-common/src/metrics/registry/mod.rs +++ b/foyer-common/src/metrics/registry/mod.rs @@ -19,6 +19,14 @@ pub mod noop; #[cfg(feature = "prometheus")] pub mod prometheus; +/// Prometheus metrics components. +#[cfg(feature = "prometheus-client")] +pub use prometheus_client_0_22 as prometheus_client; + +/// Prometheus metrics components. +#[cfg(feature = "prometheus-client_0_22")] +pub mod prometheus_client_0_22; + #[cfg(feature = "opentelemetry")] pub use opentelemetry_0_27 as opentelemetry; diff --git a/foyer-common/src/metrics/registry/noop.rs b/foyer-common/src/metrics/registry/noop.rs index 16cc4215..73199d9a 100644 --- a/foyer-common/src/metrics/registry/noop.rs +++ b/foyer-common/src/metrics/registry/noop.rs @@ -23,7 +23,7 @@ impl CounterOps for NoopMetricsRegistry { } impl CounterVecOps for NoopMetricsRegistry { - fn counter(&self, _: &[&str]) -> impl CounterOps { + fn counter(&self, _: &[&'static str]) -> impl CounterOps { NoopMetricsRegistry } } @@ -37,7 +37,7 @@ impl GaugeOps for NoopMetricsRegistry { } impl GaugeVecOps for NoopMetricsRegistry { - fn gauge(&self, _: &[&str]) -> impl GaugeOps { + fn gauge(&self, _: &[&'static str]) -> impl GaugeOps { NoopMetricsRegistry } } @@ -47,7 +47,7 @@ impl HistogramOps for NoopMetricsRegistry { } impl HistogramVecOps for NoopMetricsRegistry { - fn histogram(&self, _: &[&str]) -> impl HistogramOps { + fn histogram(&self, _: &[&'static str]) -> impl HistogramOps { NoopMetricsRegistry } } diff --git a/foyer-common/src/metrics/registry/opentelemetry_0_26.rs b/foyer-common/src/metrics/registry/opentelemetry_0_26.rs index e4374c16..26e7ef06 100644 --- a/foyer-common/src/metrics/registry/opentelemetry_0_26.rs +++ b/foyer-common/src/metrics/registry/opentelemetry_0_26.rs @@ -84,7 +84,7 @@ pub struct MetricVec { } impl CounterVecOps for MetricVec { - fn counter(&self, labels: &[&str]) -> impl CounterOps { + fn counter(&self, labels: &[&'static str]) -> impl CounterOps { let counter = self.meter.u64_counter(self.name).with_description(self.desc).init(); let labels = self .label_names @@ -97,7 +97,7 @@ impl CounterVecOps for MetricVec { } impl GaugeVecOps for MetricVec { - fn gauge(&self, labels: &[&str]) -> impl GaugeOps { + fn gauge(&self, labels: &[&'static str]) -> impl GaugeOps { let gauge = self.meter.u64_gauge(self.name).with_description(self.desc).init(); let labels = self .label_names @@ -111,7 +111,7 @@ impl GaugeVecOps for MetricVec { } impl HistogramVecOps for MetricVec { - fn histogram(&self, labels: &[&str]) -> impl HistogramOps { + fn histogram(&self, labels: &[&'static str]) -> impl HistogramOps { let histogram = self.meter.f64_histogram(self.name).with_description(self.desc).init(); let labels = self .label_names diff --git a/foyer-common/src/metrics/registry/opentelemetry_0_27.rs b/foyer-common/src/metrics/registry/opentelemetry_0_27.rs index 99ae98c1..8df31ef4 100644 --- a/foyer-common/src/metrics/registry/opentelemetry_0_27.rs +++ b/foyer-common/src/metrics/registry/opentelemetry_0_27.rs @@ -84,7 +84,7 @@ pub struct MetricVec { } impl CounterVecOps for MetricVec { - fn counter(&self, labels: &[&str]) -> impl CounterOps { + fn counter(&self, labels: &[&'static str]) -> impl CounterOps { let counter = self.meter.u64_counter(self.name).with_description(self.desc).build(); let labels = self .label_names @@ -97,7 +97,7 @@ impl CounterVecOps for MetricVec { } impl GaugeVecOps for MetricVec { - fn gauge(&self, labels: &[&str]) -> impl GaugeOps { + fn gauge(&self, labels: &[&'static str]) -> impl GaugeOps { let gauge = self.meter.u64_gauge(self.name).with_description(self.desc).build(); let labels = self .label_names @@ -111,7 +111,7 @@ impl GaugeVecOps for MetricVec { } impl HistogramVecOps for MetricVec { - fn histogram(&self, labels: &[&str]) -> impl HistogramOps { + fn histogram(&self, labels: &[&'static str]) -> impl HistogramOps { let histogram = self.meter.f64_histogram(self.name).with_description(self.desc).build(); let labels = self .label_names diff --git a/foyer-common/src/metrics/registry/prometheus.rs b/foyer-common/src/metrics/registry/prometheus.rs index fbb3fbe6..3d492264 100644 --- a/foyer-common/src/metrics/registry/prometheus.rs +++ b/foyer-common/src/metrics/registry/prometheus.rs @@ -119,7 +119,7 @@ impl CounterOps for IntCounter { } impl CounterVecOps for IntCounterVec { - fn counter(&self, labels: &[&str]) -> impl CounterOps { + fn counter(&self, labels: &[&'static str]) -> impl CounterOps { self.with_label_values(labels) } } @@ -139,7 +139,7 @@ impl GaugeOps for IntGauge { } impl GaugeVecOps for IntGaugeVec { - fn gauge(&self, labels: &[&str]) -> impl GaugeOps { + fn gauge(&self, labels: &[&'static str]) -> impl GaugeOps { self.with_label_values(labels) } } @@ -151,12 +151,12 @@ impl HistogramOps for Histogram { } impl HistogramVecOps for HistogramVec { - fn histogram(&self, labels: &[&str]) -> impl HistogramOps { + fn histogram(&self, labels: &[&'static str]) -> impl HistogramOps { self.with_label_values(labels) } } -/// Prometheus metrics registry. +/// Prometheus metric registry with lib `prometheus`. /// /// The [`PrometheusMetricsRegistry`] can be cloned and used by multiple foyer instances, without worrying about /// duplicately registering. diff --git a/foyer-common/src/metrics/registry/prometheus_client_0_22.rs b/foyer-common/src/metrics/registry/prometheus_client_0_22.rs new file mode 100644 index 00000000..bef64f12 --- /dev/null +++ b/foyer-common/src/metrics/registry/prometheus_client_0_22.rs @@ -0,0 +1,232 @@ +// Copyright 2024 foyer Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use itertools::Itertools; +use parking_lot::Mutex; +use prometheus_client_0_22::{ + encoding::{EncodeLabel, EncodeLabelSet, LabelSetEncoder}, + metrics::{ + counter::Counter as PcCounter, family::Family, gauge::Gauge as PcGauge, histogram::Histogram as PcHistogram, + }, + registry::Registry, +}; + +use crate::metrics::{CounterOps, CounterVecOps, GaugeOps, GaugeVecOps, HistogramOps, HistogramVecOps, RegistryOps}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct Labels { + pairs: Vec<(&'static str, &'static str)>, +} + +impl EncodeLabelSet for Labels { + fn encode(&self, mut encoder: LabelSetEncoder) -> Result<(), std::fmt::Error> { + for pair in self.pairs.iter() { + pair.encode(encoder.encode_label())?; + } + Ok(()) + } +} + +#[derive(Debug)] +struct Counter { + counter: Family, + labels: Labels, +} + +impl CounterOps for Counter { + fn increase(&self, val: u64) { + self.counter.get_or_create(&self.labels).inc_by(val); + } +} + +#[derive(Debug)] +struct CounterVec { + counter: Family, + label_names: &'static [&'static str], +} + +impl CounterVecOps for CounterVec { + fn counter(&self, labels: &[&'static str]) -> impl CounterOps { + Counter { + counter: self.counter.clone(), + labels: Labels { + pairs: self + .label_names + .iter() + .zip_eq(labels.iter()) + .map(|(name, label)| (*name, *label)) + .collect(), + }, + } + } +} + +#[derive(Debug)] +struct Gauge { + gauge: Family, + labels: Labels, +} + +impl GaugeOps for Gauge { + fn increase(&self, val: u64) { + self.gauge.get_or_create(&self.labels).inc_by(val as _); + } + + fn decrease(&self, val: u64) { + self.gauge.get_or_create(&self.labels).dec_by(val as _); + } + + fn absolute(&self, val: u64) { + self.gauge.get_or_create(&self.labels).set(val as _); + } +} + +#[derive(Debug)] +struct GaugeVec { + gauge: Family, + label_names: &'static [&'static str], +} + +impl GaugeVecOps for GaugeVec { + fn gauge(&self, labels: &[&'static str]) -> impl GaugeOps { + Gauge { + gauge: self.gauge.clone(), + labels: Labels { + pairs: self + .label_names + .iter() + .zip_eq(labels.iter()) + .map(|(name, label)| (*name, *label)) + .collect(), + }, + } + } +} + +#[derive(Debug)] +struct Histogram { + histogram: Family, + labels: Labels, +} + +impl HistogramOps for Histogram { + fn record(&self, val: f64) { + self.histogram.get_or_create(&self.labels).observe(val); + } +} + +#[derive(Debug)] +struct HistogramVec { + histogram: Family, + label_names: &'static [&'static str], +} + +impl HistogramVecOps for HistogramVec { + fn histogram(&self, labels: &[&'static str]) -> impl HistogramOps { + Histogram { + histogram: self.histogram.clone(), + labels: Labels { + pairs: self + .label_names + .iter() + .zip_eq(labels.iter()) + .map(|(name, label)| (*name, *label)) + .collect(), + }, + } + } +} + +#[derive(Debug, Clone)] +/// Prometheus metric registry with lib `prometheus-client`. +pub struct PrometheusClientMetricsRegistry { + registry: Arc>, +} + +impl PrometheusClientMetricsRegistry { + /// Create an Prometheus metrics registry. + pub fn new(registry: Arc>) -> Self { + Self { registry } + } +} + +impl RegistryOps for PrometheusClientMetricsRegistry { + fn register_counter_vec( + &self, + name: &'static str, + desc: &'static str, + label_names: &'static [&'static str], + ) -> impl CounterVecOps { + let counter = Family::::default(); + self.registry.lock().register(name, desc, counter.clone()); + CounterVec { counter, label_names } + } + + fn register_gauge_vec( + &self, + name: &'static str, + desc: &'static str, + label_names: &'static [&'static str], + ) -> impl GaugeVecOps { + let gauge = Family::::default(); + self.registry.lock().register(name, desc, gauge.clone()); + GaugeVec { gauge, label_names } + } + + fn register_histogram_vec( + &self, + name: &'static str, + desc: &'static str, + label_names: &'static [&'static str], + ) -> impl HistogramVecOps { + let histogram = Family::::new_with_constructor(|| { + PcHistogram::new([0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0].into_iter()) + }); + self.registry.lock().register(name, desc, histogram.clone()); + HistogramVec { histogram, label_names } + } +} + +#[cfg(test)] +mod tests { + use prometheus_client_0_22::encoding::text::encode; + + use super::*; + + #[test] + fn test() { + let registry = Arc::new(Mutex::new(Registry::default())); + let pc = PrometheusClientMetricsRegistry::new(registry.clone()); + + let cv = pc.register_counter_vec("test_counter_1", "test counter 1", &["label1", "label2"]); + let c = cv.counter(&["l1", "l2"]); + c.increase(42); + + let gv = pc.register_gauge_vec("test_gauge_1", "test gauge 1", &["label1", "label2"]); + let g = gv.gauge(&["l1", "l2"]); + g.increase(514); + g.decrease(114); + g.absolute(114514); + + let hv = pc.register_histogram_vec("test_histogram_1", "test histogram 1", &["label1", "label2"]); + let h = hv.histogram(&["l1", "l2"]); + h.record(114.514); + + let mut text = String::new(); + encode(&mut text, ®istry.lock()).unwrap(); + println!("{text}"); + } +} diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml index b61e65c2..f695652c 100644 --- a/foyer/Cargo.toml +++ b/foyer/Cargo.toml @@ -46,6 +46,8 @@ tracing = [ "foyer-storage/tracing", ] prometheus = ["foyer-common/prometheus"] +prometheus-client = ["foyer-common/prometheus-client"] +prometheus-client_0_22 = ["foyer-common/prometheus-client_0_22"] opentelemetry = ["foyer-common/opentelemetry"] opentelemetry_0_27 = ["foyer-common/opentelemetry_0_27"] opentelemetry_0_26 = ["foyer-common/opentelemetry_0_26"] diff --git a/foyer/src/prelude.rs b/foyer/src/prelude.rs index c85ab6d2..2873547b 100644 --- a/foyer/src/prelude.rs +++ b/foyer/src/prelude.rs @@ -20,6 +20,10 @@ pub use crate::common::metrics::registry::opentelemetry_0_26; pub use crate::common::metrics::registry::opentelemetry_0_27; #[cfg(feature = "prometheus")] pub use crate::common::metrics::registry::prometheus; +#[cfg(feature = "prometheus-client")] +pub use crate::common::metrics::registry::prometheus_client; +#[cfg(feature = "prometheus-client_0_22")] +pub use crate::common::metrics::registry::prometheus_client_0_22; pub use crate::{ common::{ buf::{BufExt, BufMutExt},