feat(risedev): add some configs for monitoring

risedev.yml

@@ -1142,6 +1142,9 @@ template:
     # Frontend used by this Prometheus instance
     provide-frontend: "frontend*"
 
+    # How frequently Prometheus scrape targets (collect metrics)
+    scrape-interval: 15s
+
   frontend:
     # Advertise address of frontend
     address: "127.0.0.1"
@@ -1233,6 +1236,8 @@ template:
     # gRPC listen port of the OTLP collector
     otlp-port: 4317
 
+    max-bytes-per-trace: 5000000
+
   opendal:
     id: opendal
 

src/risedevtool/src/config_gen/grafana_gen.rs

@@ -29,6 +29,8 @@ impl GrafanaGen {
 [server]
 http_addr = {grafana_host}
 http_port = {grafana_port}
+grpc-max-recv-msg-size-bytes = 104857600 # 100 Mb
+grpc-max-send-msg-size-bytes = 104857600 # 100 Mb
 
 [users]
 default_theme = light

src/risedevtool/src/config_gen/prometheus_gen.rs

@@ -23,6 +23,7 @@ impl PrometheusGen {
     pub fn gen_prometheus_yml(&self, config: &PrometheusConfig) -> String {
         let prometheus_host = &config.address;
         let prometheus_port = &config.port;
+        let scrape_interval = &config.scrape_interval;
         let compute_node_targets = config
             .provide_compute_node
             .as_ref()
@@ -104,7 +105,7 @@ remote_write:
         format!(
             r#"# --- THIS FILE IS AUTO GENERATED BY RISEDEV ---
 global:
-  scrape_interval: 15s
+  scrape_interval: {scrape_interval}
   evaluation_interval: 60s
   external_labels:
     rw_cluster: {now}

src/risedevtool/src/config_gen/tempo_gen.rs

@@ -23,19 +23,41 @@ impl TempoGen {
         let otlp_host = &config.listen_address;
         let otlp_port = config.otlp_port;
 
+        let max_bytes_per_trace = config.max_bytes_per_trace;
+
         format!(
             r#"# --- THIS FILE IS AUTO GENERATED BY RISEDEV ---
 server:
   http_listen_address: "{http_listen_address}"
   http_listen_port: {http_listen_port}
+  grpc_server_max_recv_msg_size: 104857600 # 100 Mb
+  grpc_server_max_send_msg_size: 104857600 # 100 Mb
 
 distributor:
   receivers:
       otlp:
         protocols:
           grpc:
             endpoint: "{otlp_host}:{otlp_port}"
-    "#
+
+# Overrides configuration block
+overrides:
+
+  # Global ingestion limits configurations
+  defaults:
+
+    # Global enforced overrides
+    global:
+      # Maximum size of a single trace in bytes.  A value of 0 disables the size
+      # check.
+      # This limit is used in 3 places:
+      #  - During search, traces will be skipped when they exceed this threshold.
+      #  - During ingestion, traces that exceed this threshold will be refused.
+      #  - During compaction, traces that exceed this threshold will be partially dropped.
+      # During ingestion, exceeding the threshold results in errors like
+      #    TRACE_TOO_LARGE: max size of trace (5000000) exceeded while adding 387 bytes
+      max_bytes_per_trace: {max_bytes_per_trace}
+"#
         )
     }
 }

src/risedevtool/src/service_config.rs

@@ -185,6 +185,8 @@ pub struct PrometheusConfig {
     pub remote_write_region: String,
     pub remote_write_url: String,
 
+    pub scrape_interval: String,
+
     pub provide_compute_node: Option<Vec<ComputeNodeConfig>>,
     pub provide_meta_node: Option<Vec<MetaNodeConfig>>,
     pub provide_minio: Option<Vec<MinioConfig>>,
@@ -221,6 +223,7 @@ pub struct TempoConfig {
     pub address: String,
     pub port: u16,
     pub otlp_port: u16,
+    pub max_bytes_per_trace: usize,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]

src/risedevtool/tempo.toml

@@ -3,7 +3,7 @@ extend = "common.toml"
 [env]
 TEMPO_SYSTEM = "${SYSTEM_UNDERSCORE}"
 TEMPO_DOWNLOAD_PATH = "${PREFIX_TMP}/tempo.tar.gz"
-TEMPO_VERSION = "2.1.1"
+TEMPO_VERSION = "2.3.1"
 TEMPO_RELEASE = "tempo_${TEMPO_VERSION}_${TEMPO_SYSTEM}"
 TEMPO_DOWNLOAD_TAR_GZ = "https://github.com/grafana/tempo/releases/download/v${TEMPO_VERSION}/${TEMPO_RELEASE}.tar.gz"