feat: Support export metric in remote write format

config/datanode.example.toml

@@ -101,3 +101,17 @@ parallel_scan_channel_size = 32
 # [logging]
 # dir = "/tmp/greptimedb/logs"
 # level = "info"
+
+# datanode export the metrics generated by itself
+# encoded to Prometheus remote-write format
+# and send to Prometheus remote-write compatible receiver (e.g. send to `greptimedb` itself)
+# This is only used for `greptimedb` to export its own metric internally. Please see `logging` option for normal export of metric.
+# [system_metric]
+# whether enable export system_metric, default is false
+# enable = false
+# The url of metric export endpoint, default is `greptimedb` default frontend endpoint
+# endpoint = "127.0.0.1:4000"
+# The database name of exported metrics stores, user needs to specify a valid database
+# db = ""
+# The interval of export metric
+# write_interval = "30s"

config/frontend.example.toml

@@ -77,3 +77,17 @@ tcp_nodelay = true
 timeout = "10s"
 connect_timeout = "10s"
 tcp_nodelay = true
+
+# frontend export the metrics generated by itself
+# encoded to Prometheus remote-write format
+# and send to Prometheus remote-write compatible receiver (e.g. send to `greptimedb` itself)
+# This is only used for `greptimedb` to export its own metric internally. Please see `logging` option for normal export of metric.
+# [system_metric]
+# whether enable export system_metric, default is false
+# enable = false
+# The url of metric export endpoint, default is `greptimedb` default frontend endpoint
+# endpoint = "127.0.0.1:4000"
+# The database name of exported metrics stores, user needs to specify a valid database
+# db = ""
+# The interval of export metric
+# write_interval = "30s"

config/metasrv.example.toml

@@ -66,3 +66,17 @@ provider = "raft_engine"
 # num_partitions = 1
 # Expected number of replicas of each partition.
 # replication_factor = 3
+
+# metasrv export the metrics generated by itself
+# encoded to Prometheus remote-write format
+# and send to Prometheus remote-write compatible receiver (e.g. send to `greptimedb` itself)
+# This is only used for `greptimedb` to export its own metric internally. Please see `logging` option for normal export of metric.
+# [system_metric]
+# whether enable export system_metric, default is false
+# enable = false
+# The url of metric export endpoint, default is `greptimedb` default frontend endpoint
+# endpoint = "127.0.0.1:4000"
+# The database name of exported metrics stores, user needs to specify a valid database
+# db = ""
+# The interval of export metric
+# write_interval = "30s"

config/standalone.example.toml

@@ -177,3 +177,17 @@ parallel_scan_channel_size = 32
 # otlp_endpoint = "localhost:4317"
 # The percentage of tracing will be sampled and exported. Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1. ratio > 1 are treated as 1. Fractions < 0 are treated as 0
 # tracing_sample_ratio = 1.0
+
+# standalone export the metrics generated by itself
+# encoded to Prometheus remote-write format
+# and send to Prometheus remote-write compatible receiver (e.g. send to `greptimedb` itself)
+# This is only used for `greptimedb` to export its own metric internally. Please see `logging` option for normal export of metric.
+# [system_metric]
+# whether enable export system_metric, default is false
+# enable = false
+# The url of metric export endpoint, default is `greptimedb` default frontend endpoint
+# endpoint = "127.0.0.1:4000"
+# The database name of exported metrics stores, user needs to specify a valid database
+# db = ""
+# The interval of export metric
+# write_interval = "30s"

src/cmd/src/frontend.rs

@@ -249,6 +249,10 @@ impl StartCommand {
         .await
         .context(StartFrontendSnafu)?;
 
+        instance
+            .build_system_metric_task(&opts.system_metric)
+            .context(StartFrontendSnafu)?;
+
         instance
             .build_servers(opts)
             .await

src/cmd/src/standalone.rs

@@ -45,6 +45,7 @@ use frontend::service_config::{
 use mito2::config::MitoConfig;
 use serde::{Deserialize, Serialize};
 use servers::http::HttpOptions;
+use servers::system_metric::SystemMetricOption;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
 use snafu::ResultExt;
@@ -112,6 +113,7 @@ pub struct StandaloneOptions {
     pub user_provider: Option<String>,
     /// Options for different store engines.
     pub region_engine: Vec<RegionEngineConfig>,
+    pub system_metric: SystemMetricOption,
 }
 
 impl Default for StandaloneOptions {
@@ -131,6 +133,7 @@ impl Default for StandaloneOptions {
             metadata_store: KvBackendConfig::default(),
             procedure: ProcedureConfig::default(),
             logging: LoggingOptions::default(),
+            system_metric: SystemMetricOption::default(),
             user_provider: None,
             region_engine: vec![
                 RegionEngineConfig::Mito(MitoConfig::default()),
@@ -154,6 +157,8 @@ impl StandaloneOptions {
             meta_client: None,
             logging: self.logging,
             user_provider: self.user_provider,
+            // Handle the system metric task run by standalone to frontend for execution
+            system_metric: self.system_metric,
             ..Default::default()
         }
     }
@@ -399,6 +404,10 @@ impl StartCommand {
             .await
             .context(StartFrontendSnafu)?;
 
+        frontend
+            .build_system_metric_task(&opts.frontend.system_metric)
+            .context(StartFrontendSnafu)?;
+
         frontend
             .build_servers(opts)
             .await

src/common/telemetry/Cargo.toml

@@ -12,6 +12,7 @@ deadlock_detection = ["parking_lot/deadlock_detection"]
 backtrace = "0.3"
 common-error.workspace = true
 console-subscriber = { version = "0.1", optional = true }
+greptime-proto.workspace = true
 lazy_static.workspace = true
 once_cell.workspace = true
 opentelemetry = { version = "0.21.0", default-features = false, features = [