From f920e6c12cdb512e4a5465d91428f0f23e361a3f Mon Sep 17 00:00:00 2001 From: Oliver Gould Date: Tue, 5 Dec 2023 15:15:25 -0800 Subject: [PATCH] metrics: Use prometheus-client for proxy_build_info (#2551) In preparation to add metrics that report float values, and ultimately in service of replacing Linkerd's custom metrics implementation with an upstream library, this change adds an integration with the prometheus-client crate. This is accomplished by leaving the existing types and traits in place and adding a FmtMetrics implementation for the prometheus-client Registry type, minimizing the change needed to support this new library. New metrics can use the upstream registry and metric types. The proxy_build_info metric has been ported to use the new API to validate the new library. Now, the metrics dump ends with: # HELP proxy_build_info Proxy build info. # TYPE proxy_build_info gauge proxy_build_info{date="2023-12-05T19:52:40Z",git_sha="215c32d3f",profile="release",vendor="code@ver-sea",version="0.0.0-dev.215c32d3f"} 1 # EOF --- Cargo.lock | 31 ++++++++++++++++ linkerd/app/core/Cargo.toml | 1 + linkerd/app/core/src/build_info.rs | 28 +++++++++++++++ linkerd/app/core/src/lib.rs | 2 ++ linkerd/app/core/src/metrics.rs | 28 ++++++++++----- linkerd/app/core/src/telemetry.rs | 2 +- linkerd/app/core/src/telemetry/build_info.rs | 38 -------------------- linkerd/metrics/Cargo.toml | 1 + linkerd/metrics/src/counter.rs | 2 +- linkerd/metrics/src/{prom.rs => fmt.rs} | 0 linkerd/metrics/src/gauge.rs | 2 +- linkerd/metrics/src/lib.rs | 33 +++++++++++++++-- linkerd2-proxy/src/main.rs | 15 ++++---- 13 files changed, 123 insertions(+), 60 deletions(-) create mode 100644 linkerd/app/core/src/build_info.rs delete mode 100644 linkerd/app/core/src/telemetry/build_info.rs rename linkerd/metrics/src/{prom.rs => fmt.rs} (100%) diff --git a/Cargo.lock b/Cargo.lock index ed4ed939c6..439d67a079 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -465,6 +465,12 @@ dependencies = [ "tower", ] +[[package]] +name = "dtoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" + [[package]] name = "either" version = "1.8.1" @@ -1143,6 +1149,7 @@ dependencies = [ "linkerd-transport-metrics", "parking_lot", "pin-project", + "prometheus-client", "quickcheck", "regex", "semver", @@ -1623,6 +1630,7 @@ dependencies = [ "hyper", "linkerd-stack", "parking_lot", + "prometheus-client", "quickcheck", "tokio", "tracing", @@ -2550,6 +2558,29 @@ dependencies = [ "rustix 0.36.16", ] +[[package]] +name = "prometheus-client" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "510c4f1c9d81d556458f94c98f857748130ea9737bbd6053da497503b26ea63c" +dependencies = [ + "dtoa", + "itoa", + "parking_lot", + "prometheus-client-derive-encode", +] + +[[package]] +name = "prometheus-client-derive-encode" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "prost" version = "0.12.1" diff --git a/linkerd/app/core/Cargo.toml b/linkerd/app/core/Cargo.toml index f620e18c52..30e22b0dc1 100644 --- a/linkerd/app/core/Cargo.toml +++ b/linkerd/app/core/Cargo.toml @@ -58,6 +58,7 @@ linkerd-transport-header = { path = "../../transport-header" } linkerd-transport-metrics = { path = "../../transport-metrics" } linkerd-tls = { path = "../../tls" } linkerd-trace-context = { path = "../../trace-context" } +prometheus-client = "0.22" regex = "1" serde_json = "1" thiserror = "1" diff --git a/linkerd/app/core/src/build_info.rs b/linkerd/app/core/src/build_info.rs new file mode 100644 index 0000000000..66b19329c7 --- /dev/null +++ b/linkerd/app/core/src/build_info.rs @@ -0,0 +1,28 @@ +use linkerd_metrics::prom::{self, encoding::EncodeLabelSet}; + +pub const BUILD_INFO: BuildInfo = BuildInfo { + date: env!("LINKERD2_PROXY_BUILD_DATE"), + git_sha: env!("GIT_SHA"), + profile: env!("PROFILE"), + vendor: env!("LINKERD2_PROXY_VENDOR"), + version: env!("LINKERD2_PROXY_VERSION"), +}; + +#[derive(Copy, Clone, Debug, Default, Hash, PartialEq, Eq, EncodeLabelSet)] +pub struct BuildInfo { + pub date: &'static str, + pub git_sha: &'static str, + pub profile: &'static str, + pub vendor: &'static str, + pub version: &'static str, +} + +impl BuildInfo { + pub fn metric(&self) -> prom::Family { + let fam = prom::Family::::new_with_constructor(|| { + prom::ConstGauge::new(1) + }); + let _ = fam.get_or_create(self); + fam + } +} diff --git a/linkerd/app/core/src/lib.rs b/linkerd/app/core/src/lib.rs index 3141a88e1d..a10a0d9721 100644 --- a/linkerd/app/core/src/lib.rs +++ b/linkerd/app/core/src/lib.rs @@ -13,6 +13,7 @@ use thiserror::Error; +mod build_info; pub mod classify; pub mod config; pub mod control; @@ -27,6 +28,7 @@ pub mod svc; pub mod telemetry; pub mod transport; +pub use self::build_info::{BuildInfo, BUILD_INFO}; pub use drain; pub use ipnet::{IpNet, Ipv4Net, Ipv6Net}; pub use linkerd_addr::{self as addr, Addr, AddrMatch, IpMatch, NameAddr, NameMatch}; diff --git a/linkerd/app/core/src/metrics.rs b/linkerd/app/core/src/metrics.rs index ffca10efca..2079271b83 100644 --- a/linkerd/app/core/src/metrics.rs +++ b/linkerd/app/core/src/metrics.rs @@ -9,7 +9,7 @@ pub use crate::transport::labels::{TargetAddr, TlsAccept}; use crate::{ classify::Class, - control, http_metrics, http_metrics as metrics, opencensus, profiles, stack_metrics, + control, http_metrics, opencensus, profiles, stack_metrics, svc::Param, telemetry, tls, transport::{self, labels::TlsConnect}, @@ -39,6 +39,9 @@ pub struct Metrics { pub proxy: Proxy, pub control: ControlHttp, pub opencensus: opencensus::metrics::Registry, + + // Global prometheus registry for new metrics. + pub registry: prom::Registry, } #[derive(Clone, Debug)] @@ -147,36 +150,42 @@ impl Metrics { retain_idle: Duration, start_time: telemetry::StartTime, ) -> (Self, impl FmtMetrics + Clone + Send + 'static) { + let registry = prom::Registry::default(); + let process = telemetry::process::Report::new(start_time); - let build_info = telemetry::build_info::Report::default(); + registry.write().register( + "proxy_build_info", + "Proxy build info", + crate::BUILD_INFO.metric(), + ); let (control, control_report) = { - let m = metrics::Requests::::default(); + let m = http_metrics::Requests::::default(); let r = m.clone().into_report(retain_idle).with_prefix("control"); (m, r) }; let (http_endpoint, endpoint_report) = { - let m = metrics::Requests::::default(); + let m = http_metrics::Requests::::default(); let r = m.clone().into_report(retain_idle); (m, r) }; let (http_profile_route, profile_route_report) = { - let m = metrics::Requests::::default(); + let m = http_metrics::Requests::::default(); let r = m.clone().into_report(retain_idle).with_prefix("route"); (m, r) }; let (http_profile_route_retry, retry_report) = { - let m = metrics::Retries::::default(); + let m = http_metrics::Retries::::default(); let r = m.clone().into_report(retain_idle).with_prefix("route"); (m, r) }; let (http_profile_route_actual, actual_report) = { - let m = metrics::Requests::::default(); + let m = http_metrics::Requests::::default(); let r = m .clone() .into_report(retain_idle) @@ -203,6 +212,7 @@ impl Metrics { proxy, control, opencensus, + registry: registry.clone(), }; let report = endpoint_report @@ -214,7 +224,9 @@ impl Metrics { .and_report(opencensus_report) .and_report(stack) .and_report(process) - .and_report(build_info); + // The prom registry reports an "# EOF" at the end of its export, so + // it should be emitted last. + .and_report(registry); (metrics, report) } diff --git a/linkerd/app/core/src/telemetry.rs b/linkerd/app/core/src/telemetry.rs index 63acd33a92..6516165c0a 100644 --- a/linkerd/app/core/src/telemetry.rs +++ b/linkerd/app/core/src/telemetry.rs @@ -1,3 +1,3 @@ -pub mod build_info; pub mod process; + pub use self::process::StartTime; diff --git a/linkerd/app/core/src/telemetry/build_info.rs b/linkerd/app/core/src/telemetry/build_info.rs deleted file mode 100644 index 5e96de5aad..0000000000 --- a/linkerd/app/core/src/telemetry/build_info.rs +++ /dev/null @@ -1,38 +0,0 @@ -use linkerd_metrics::{metrics, FmtLabels, FmtMetric, FmtMetrics, Gauge}; -use std::{env, fmt}; - -pub const VERSION: &str = env!("LINKERD2_PROXY_VERSION"); -pub const DATE: &str = env!("LINKERD2_PROXY_BUILD_DATE"); -pub const VENDOR: &str = env!("LINKERD2_PROXY_VENDOR"); -pub const GIT_SHA: &str = env!("GIT_SHA"); -pub const PROFILE: &str = env!("PROFILE"); - -metrics! { - proxy_build_info: Gauge { - "Proxy build info" - } -} - -#[derive(Clone, Debug, Default)] -pub struct Report(()); - -struct Labels; - -impl FmtMetrics for Report { - fn fmt_metrics(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - proxy_build_info.fmt_help(f)?; - Gauge::from(1).fmt_metric_labeled(f, "proxy_build_info", &Labels)?; - Ok(()) - } -} - -impl FmtLabels for Labels { - fn fmt_labels(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "version=\"{VERSION}\"")?; - write!(f, ",git_sha=\"{GIT_SHA}\"")?; - write!(f, ",profile=\"{PROFILE}\"")?; - write!(f, ",date=\"{DATE}\"")?; - write!(f, ",vendor=\"{VENDOR}\"")?; - Ok(()) - } -} diff --git a/linkerd/metrics/Cargo.toml b/linkerd/metrics/Cargo.toml index df83bbdddc..1cef35b5f2 100644 --- a/linkerd/metrics/Cargo.toml +++ b/linkerd/metrics/Cargo.toml @@ -16,6 +16,7 @@ http = "0.2" hyper = { version = "0.14", features = ["http1", "http2"] } linkerd-stack = { path = "../stack", optional = true } parking_lot = "0.12" +prometheus-client = "0.22" tokio = { version = "1", features = ["time"] } tracing = "0.1" diff --git a/linkerd/metrics/src/counter.rs b/linkerd/metrics/src/counter.rs index 9513c3df13..e9f037e697 100644 --- a/linkerd/metrics/src/counter.rs +++ b/linkerd/metrics/src/counter.rs @@ -1,5 +1,5 @@ use super::{ - prom::{FmtLabels, FmtMetric}, + fmt::{FmtLabels, FmtMetric}, Factor, }; use std::fmt::{self, Display}; diff --git a/linkerd/metrics/src/prom.rs b/linkerd/metrics/src/fmt.rs similarity index 100% rename from linkerd/metrics/src/prom.rs rename to linkerd/metrics/src/fmt.rs diff --git a/linkerd/metrics/src/gauge.rs b/linkerd/metrics/src/gauge.rs index d35b3416c6..63952477e5 100644 --- a/linkerd/metrics/src/gauge.rs +++ b/linkerd/metrics/src/gauge.rs @@ -1,4 +1,4 @@ -use super::prom::{FmtLabels, FmtMetric}; +use super::fmt::{FmtLabels, FmtMetric}; use std::fmt::{self, Display}; use std::sync::atomic::{AtomicU64, Ordering}; diff --git a/linkerd/metrics/src/lib.rs b/linkerd/metrics/src/lib.rs index ed717e709e..a0a208aa57 100644 --- a/linkerd/metrics/src/lib.rs +++ b/linkerd/metrics/src/lib.rs @@ -4,12 +4,12 @@ //! Utilities for exposing metrics to Prometheus. mod counter; +mod fmt; mod gauge; mod histogram; pub mod latency; #[cfg(feature = "linkerd-stack")] mod new_metrics; -mod prom; mod serve; mod store; @@ -17,13 +17,42 @@ mod store; pub use self::new_metrics::NewMetrics; pub use self::{ counter::Counter, + fmt::{FmtLabels, FmtMetric, FmtMetrics, Metric}, gauge::Gauge, histogram::Histogram, - prom::{FmtLabels, FmtMetric, FmtMetrics, Metric}, serve::Serve, store::{LastUpdate, SharedStore, Store}, }; +/// Integration with the [`prometheus_client`]` crate. +/// +/// This should be used for all new metrics. +pub mod prom { + use parking_lot::RwLock; + use std::sync::Arc; + + pub use prometheus_client::{ + metrics::{ + counter::{ConstCounter, Counter}, + family::Family, + gauge::{ConstGauge, Gauge}, + histogram::Histogram, + info::Info, + }, + *, + }; + + /// New metrics should use the prometheus-client Registry. + pub type Registry = Arc>; + + impl crate::FmtMetrics for Registry { + #[inline] + fn fmt_metrics(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + encoding::text::encode(f, &self.read()) + } + } +} + #[macro_export] macro_rules! metrics { { $( $name:ident : $kind:ty { $help:expr } ),+ } => { diff --git a/linkerd2-proxy/src/main.rs b/linkerd2-proxy/src/main.rs index 2a2a0c7aa2..499edd5797 100644 --- a/linkerd2-proxy/src/main.rs +++ b/linkerd2-proxy/src/main.rs @@ -12,10 +12,7 @@ compile_error!( ); use linkerd_app::{ - core::{ - telemetry::{build_info, StartTime}, - transport::BindTcp, - }, + core::{telemetry::StartTime, transport::BindTcp, BUILD_INFO}, trace, Config, }; use linkerd_signal as signal; @@ -42,11 +39,11 @@ fn main() { info!( "{profile} {version} ({sha}) by {vendor} on {date}", - date = build_info::DATE, - sha = build_info::GIT_SHA, - version = build_info::VERSION, - profile = build_info::PROFILE, - vendor = build_info::VENDOR, + date = BUILD_INFO.date, + sha = BUILD_INFO.git_sha, + version = BUILD_INFO.version, + profile = BUILD_INFO.profile, + vendor = BUILD_INFO.vendor, ); // Load configuration from the environment without binding ports.