feat: export runtime metric to promethues (#2985)

* feat: export runtime metric to promethues

* fix: export tokio metric should enable tokio_unstable

* chore: tokio-metric export add more check info

src/common/runtime/Cargo.toml

@@ -14,6 +14,8 @@ once_cell.workspace = true
 paste.workspace = true
 prometheus.workspace = true
 snafu.workspace = true
+tokio-metrics = "0.3"
+tokio-metrics-collector = "0.2"
 tokio-util.workspace = true
 tokio.workspace = true
 

src/common/runtime/src/runtime.rs

@@ -152,15 +152,19 @@ impl Builder {
             .build()
             .context(BuildRuntimeSnafu)?;
 
+        let name = self.runtime_name.clone();
         let handle = runtime.handle().clone();
         let (send_stop, recv_stop) = oneshot::channel();
         // Block the runtime to shutdown.
         let _ = thread::Builder::new()
             .name(format!("{}-blocker", self.thread_name))
             .spawn(move || runtime.block_on(recv_stop));
 
+        #[cfg(tokio_unstable)]
+        register_collector(name.clone(), &handle);
+
         Ok(Runtime {
-            name: self.runtime_name.clone(),
+            name,
             handle,
             _dropper: Arc::new(Dropper {
                 close: Some(send_stop),
@@ -169,6 +173,14 @@ impl Builder {
     }
 }
 
+#[cfg(tokio_unstable)]
+pub fn register_collector(name: String, handle: &Handle) {
+    let name = name.replace("-", "_");
+    let monitor = tokio_metrics::RuntimeMonitor::new(handle);
+    let collector = tokio_metrics_collector::RuntimeCollector::new(monitor, name);
+    let _ = prometheus::register(Box::new(collector));
+}
+
 fn on_thread_start(thread_name: String) -> impl Fn() + 'static {
     move || {
         METRIC_RUNTIME_THREADS_ALIVE
@@ -241,6 +253,13 @@ mod tests {
 
         assert!(metric_text.contains("runtime_threads_idle{thread_name=\"test_runtime_metric\"}"));
         assert!(metric_text.contains("runtime_threads_alive{thread_name=\"test_runtime_metric\"}"));
+
+        #[cfg(tokio_unstable)]
+        {
+            assert!(metric_text.contains("runtime_0_tokio_budget_forced_yield_count 0"));
+            assert!(metric_text.contains("runtime_0_tokio_injection_queue_depth 0"));
+            assert!(metric_text.contains("runtime_0_tokio_workers_count 5"));
+        }
     }
 
     #[test]