fix: replace opendal PrometheusLayer (#2861)

* fix: replace opendal `PrometheusLayer`

* chore: add docs on `PrometheusMetricsLayer`

* chore: fix code advice

* chore: fix bug on `PrometheusMetricsLayer`

src/datanode/Cargo.toml

@@ -4,9 +4,6 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 
-[features]
-testing = []
-
 [dependencies]
 api.workspace = true
 arrow-flight.workspace = true

src/datanode/src/store.rs

@@ -66,23 +66,9 @@ pub(crate) async fn new_object_store(opts: &DatanodeOptions) -> Result<ObjectSto
                 .with_error_level(Some("debug"))
                 .expect("input error level must be valid"),
         )
-        .layer(TracingLayer);
-
-    // In the test environment, multiple datanodes will be started in the same process.
-    // If each datanode registers Prometheus metric when it starts, it will cause the program to crash. (Because the same metric is registered repeatedly.)
-    // So the Prometheus metric layer is disabled in the test environment.
-    #[cfg(feature = "testing")]
-    return Ok(store);
-
-    #[cfg(not(feature = "testing"))]
-    {
-        let registry = prometheus::default_registry();
-        Ok(
-            store.layer(object_store::layers::PrometheusLayer::with_registry(
-                registry.clone(),
-            )),
-        )
-    }
+        .layer(TracingLayer)
+        .layer(object_store::layers::PrometheusMetricsLayer);
+    Ok(store)
 }
 
 async fn create_object_store_with_cache(

src/object-store/Cargo.toml

@@ -17,7 +17,6 @@ md5 = "0.7"
 moka = { workspace = true, features = ["future"] }
 opendal = { version = "0.40", features = [
     "layers-tracing",
-    "layers-prometheus",
 ] }
 prometheus.workspace = true
 snafu.workspace = true

src/object-store/src/layers.rs

@@ -13,6 +13,8 @@
 // limitations under the License.
 
 mod lru_cache;
+mod prometheus;
 
 pub use lru_cache::*;
 pub use opendal::layers::*;
+pub use prometheus::PrometheusMetricsLayer;