diff --git a/quickwit/quickwit-control-plane/src/control_plane.rs b/quickwit/quickwit-control-plane/src/control_plane.rs index 823ea42a9eb..76ac9e38303 100644 --- a/quickwit/quickwit-control-plane/src/control_plane.rs +++ b/quickwit/quickwit-control-plane/src/control_plane.rs @@ -32,8 +32,9 @@ use quickwit_config::SourceConfig; use quickwit_ingest::{IngesterPool, LocalShardsUpdate}; use quickwit_metastore::IndexMetadata; use quickwit_proto::control_plane::{ - ControlPlaneError, ControlPlaneResult, GetOrCreateOpenShardsRequest, - GetOrCreateOpenShardsResponse, + ControlPlaneError, ControlPlaneResult, GetDebugStateRequest, GetDebugStateResponse, + GetOrCreateOpenShardsRequest, GetOrCreateOpenShardsResponse, PhysicalIndexingPlanEntry, + ShardTableEntry, }; use quickwit_proto::indexing::ShardPositionsUpdate; use quickwit_proto::metastore::{ @@ -179,6 +180,37 @@ impl ControlPlane { .schedule_indexing_plan_if_needed(&self.model); Ok(()) } + + fn debug_state(&self) -> GetDebugStateResponse { + let shard_table = self + .model + .all_shards_with_source() + .map(|(source, shards)| ShardTableEntry { + source_id: source.to_string(), + shards: shards + .map(|shard_entry| shard_entry.shard.clone()) + .collect(), + }) + .collect(); + let physical_index_plan = self + .indexing_scheduler + .observable_state() + .last_applied_physical_plan + .map(|plan| { + plan.indexing_tasks_per_indexer() + .iter() + .map(|(node_id, tasks)| PhysicalIndexingPlanEntry { + node_id: node_id.clone(), + tasks: tasks.clone(), + }) + .collect() + }) + .unwrap_or_default(); + GetDebugStateResponse { + shard_table, + physical_index_plan, + } + } } #[async_trait] @@ -528,6 +560,19 @@ impl Handler for ControlPlane { } } +#[async_trait] +impl Handler for ControlPlane { + type Reply = ControlPlaneResult; + + async fn handle( + &mut self, + _: GetDebugStateRequest, + _ctx: &ActorContext, + ) -> Result { + Ok(Ok(self.debug_state())) + } +} + #[derive(Clone)] pub struct ControlPlaneEventSubscriber(WeakMailbox); diff --git a/quickwit/quickwit-control-plane/src/model/mod.rs b/quickwit/quickwit-control-plane/src/model/mod.rs index aee54dcc9d4..6c07b7adb88 100644 --- a/quickwit/quickwit-control-plane/src/model/mod.rs +++ b/quickwit/quickwit-control-plane/src/model/mod.rs @@ -248,6 +248,12 @@ impl ControlPlaneModel { self.shard_table.all_shards() } + pub(crate) fn all_shards_with_source( + &self, + ) -> impl Iterator)> + '_ { + self.shard_table.all_shards_with_source() + } + pub fn list_shards_for_node( &self, ingester: &NodeId, diff --git a/quickwit/quickwit-control-plane/src/model/shard_table.rs b/quickwit/quickwit-control-plane/src/model/shard_table.rs index 0f28de7e5ab..8ea8d8c6c76 100644 --- a/quickwit/quickwit-control-plane/src/model/shard_table.rs +++ b/quickwit/quickwit-control-plane/src/model/shard_table.rs @@ -284,6 +284,14 @@ impl ShardTable { .flat_map(|table_entry| table_entry.shard_entries.values()) } + pub(crate) fn all_shards_with_source( + &self, + ) -> impl Iterator)> + '_ { + self.table_entries + .iter() + .map(|(source, shard_table)| (source, shard_table.shard_entries.values())) + } + pub(crate) fn all_shards_mut(&mut self) -> impl Iterator + '_ { self.table_entries .values_mut() diff --git a/quickwit/quickwit-proto/protos/quickwit/control_plane.proto b/quickwit/quickwit-proto/protos/quickwit/control_plane.proto index b01776e3376..ce9d8215312 100644 --- a/quickwit/quickwit-proto/protos/quickwit/control_plane.proto +++ b/quickwit/quickwit-proto/protos/quickwit/control_plane.proto @@ -21,6 +21,7 @@ syntax = "proto3"; package quickwit.control_plane; +import "quickwit/indexing.proto"; import "quickwit/ingest.proto"; import "quickwit/metastore.proto"; @@ -59,6 +60,9 @@ service ControlPlaneService { // Returns the list of open shards for one or several sources. If the control plane is not able to find any // for a source, it will pick a pair of leader-follower ingesters and will open a new shard. rpc GetOrCreateOpenShards(GetOrCreateOpenShardsRequest) returns (GetOrCreateOpenShardsResponse); + + // Return some innerstate of the control plane meant to assist debugging. + rpc GetDebugState(GetDebugStateRequest) returns (GetDebugStateResponse); } // Shard API @@ -103,3 +107,21 @@ message GetOrCreateOpenShardsFailure { string source_id = 3; GetOrCreateOpenShardsFailureReason reason = 4; } + +message GetDebugStateRequest { +} + +message GetDebugStateResponse { + repeated ShardTableEntry shard_table = 1; + repeated PhysicalIndexingPlanEntry physical_index_plan = 2; +} + +message ShardTableEntry { + string source_id = 1; + repeated quickwit.ingest.Shard shards = 2; +} + +message PhysicalIndexingPlanEntry { + string node_id = 1; + repeated quickwit.indexing.IndexingTask tasks = 2; +} diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs index 9ccea332b54..d6ff18bb8e2 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs @@ -60,6 +60,37 @@ pub struct GetOrCreateOpenShardsFailure { pub reason: i32, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetDebugStateRequest {} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetDebugStateResponse { + #[prost(message, repeated, tag = "1")] + pub shard_table: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "2")] + pub physical_index_plan: ::prost::alloc::vec::Vec, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ShardTableEntry { + #[prost(string, tag = "1")] + pub source_id: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub shards: ::prost::alloc::vec::Vec, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PhysicalIndexingPlanEntry { + #[prost(string, tag = "1")] + pub node_id: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub tasks: ::prost::alloc::vec::Vec, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] @@ -140,6 +171,11 @@ pub trait ControlPlaneService: std::fmt::Debug + dyn_clone::DynClone + Send + Sy &mut self, request: GetOrCreateOpenShardsRequest, ) -> crate::control_plane::ControlPlaneResult; + /// Return some innerstate of the control plane meant to assist debugging. + async fn get_debug_state( + &mut self, + request: GetDebugStateRequest, + ) -> crate::control_plane::ControlPlaneResult; } dyn_clone::clone_trait_object!(ControlPlaneService); #[cfg(any(test, feature = "testsuite"))] @@ -260,6 +296,12 @@ impl ControlPlaneService for ControlPlaneServiceClient { ) -> crate::control_plane::ControlPlaneResult { self.inner.get_or_create_open_shards(request).await } + async fn get_debug_state( + &mut self, + request: GetDebugStateRequest, + ) -> crate::control_plane::ControlPlaneResult { + self.inner.get_debug_state(request).await + } } #[cfg(any(test, feature = "testsuite"))] pub mod control_plane_service_mock { @@ -318,6 +360,12 @@ pub mod control_plane_service_mock { > { self.inner.lock().await.get_or_create_open_shards(request).await } + async fn get_debug_state( + &mut self, + request: super::GetDebugStateRequest, + ) -> crate::control_plane::ControlPlaneResult { + self.inner.lock().await.get_debug_state(request).await + } } impl From for ControlPlaneServiceClient { fn from(mock: MockControlPlaneService) -> Self { @@ -432,6 +480,22 @@ impl tower::Service for Box for Box { + type Response = GetDebugStateResponse; + type Error = crate::control_plane::ControlPlaneError; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + fn call(&mut self, request: GetDebugStateRequest) -> Self::Future { + let mut svc = self.clone(); + let fut = async move { svc.get_debug_state(request).await }; + Box::pin(fut) + } +} /// A tower service stack is a set of tower services. #[derive(Debug)] struct ControlPlaneServiceTowerServiceStack { @@ -466,6 +530,11 @@ struct ControlPlaneServiceTowerServiceStack { GetOrCreateOpenShardsResponse, crate::control_plane::ControlPlaneError, >, + get_debug_state_svc: quickwit_common::tower::BoxService< + GetDebugStateRequest, + GetDebugStateResponse, + crate::control_plane::ControlPlaneError, + >, } impl Clone for ControlPlaneServiceTowerServiceStack { fn clone(&self) -> Self { @@ -477,6 +546,7 @@ impl Clone for ControlPlaneServiceTowerServiceStack { toggle_source_svc: self.toggle_source_svc.clone(), delete_source_svc: self.delete_source_svc.clone(), get_or_create_open_shards_svc: self.get_or_create_open_shards_svc.clone(), + get_debug_state_svc: self.get_debug_state_svc.clone(), } } } @@ -520,6 +590,12 @@ impl ControlPlaneService for ControlPlaneServiceTowerServiceStack { ) -> crate::control_plane::ControlPlaneResult { self.get_or_create_open_shards_svc.ready().await?.call(request).await } + async fn get_debug_state( + &mut self, + request: GetDebugStateRequest, + ) -> crate::control_plane::ControlPlaneResult { + self.get_debug_state_svc.ready().await?.call(request).await + } } type CreateIndexLayer = quickwit_common::tower::BoxLayer< quickwit_common::tower::BoxService< @@ -581,6 +657,16 @@ type GetOrCreateOpenShardsLayer = quickwit_common::tower::BoxLayer< GetOrCreateOpenShardsResponse, crate::control_plane::ControlPlaneError, >; +type GetDebugStateLayer = quickwit_common::tower::BoxLayer< + quickwit_common::tower::BoxService< + GetDebugStateRequest, + GetDebugStateResponse, + crate::control_plane::ControlPlaneError, + >, + GetDebugStateRequest, + GetDebugStateResponse, + crate::control_plane::ControlPlaneError, +>; #[derive(Debug, Default)] pub struct ControlPlaneServiceTowerLayerStack { create_index_layers: Vec, @@ -589,6 +675,7 @@ pub struct ControlPlaneServiceTowerLayerStack { toggle_source_layers: Vec, delete_source_layers: Vec, get_or_create_open_shards_layers: Vec, + get_debug_state_layers: Vec, } impl ControlPlaneServiceTowerLayerStack { pub fn stack_layer(mut self, layer: L) -> Self @@ -755,6 +842,31 @@ impl ControlPlaneServiceTowerLayerStack { >>::Service as tower::Service< GetOrCreateOpenShardsRequest, >>::Future: Send + 'static, + L: tower::Layer< + quickwit_common::tower::BoxService< + GetDebugStateRequest, + GetDebugStateResponse, + crate::control_plane::ControlPlaneError, + >, + > + Clone + Send + Sync + 'static, + , + >>::Service: tower::Service< + GetDebugStateRequest, + Response = GetDebugStateResponse, + Error = crate::control_plane::ControlPlaneError, + > + Clone + Send + Sync + 'static, + <, + >>::Service as tower::Service>::Future: Send + 'static, { self.create_index_layers .push(quickwit_common::tower::BoxLayer::new(layer.clone())); @@ -768,6 +880,8 @@ impl ControlPlaneServiceTowerLayerStack { .push(quickwit_common::tower::BoxLayer::new(layer.clone())); self.get_or_create_open_shards_layers .push(quickwit_common::tower::BoxLayer::new(layer.clone())); + self.get_debug_state_layers + .push(quickwit_common::tower::BoxLayer::new(layer.clone())); self } pub fn stack_create_index_layer(mut self, layer: L) -> Self @@ -897,6 +1011,25 @@ impl ControlPlaneServiceTowerLayerStack { .push(quickwit_common::tower::BoxLayer::new(layer)); self } + pub fn stack_get_debug_state_layer(mut self, layer: L) -> Self + where + L: tower::Layer< + quickwit_common::tower::BoxService< + GetDebugStateRequest, + GetDebugStateResponse, + crate::control_plane::ControlPlaneError, + >, + > + Send + Sync + 'static, + L::Service: tower::Service< + GetDebugStateRequest, + Response = GetDebugStateResponse, + Error = crate::control_plane::ControlPlaneError, + > + Clone + Send + Sync + 'static, + >::Future: Send + 'static, + { + self.get_debug_state_layers.push(quickwit_common::tower::BoxLayer::new(layer)); + self + } pub fn build(self, instance: T) -> ControlPlaneServiceClient where T: ControlPlaneService, @@ -982,6 +1115,14 @@ impl ControlPlaneServiceTowerLayerStack { quickwit_common::tower::BoxService::new(boxed_instance.clone()), |svc, layer| layer.layer(svc), ); + let get_debug_state_svc = self + .get_debug_state_layers + .into_iter() + .rev() + .fold( + quickwit_common::tower::BoxService::new(boxed_instance.clone()), + |svc, layer| layer.layer(svc), + ); let tower_svc_stack = ControlPlaneServiceTowerServiceStack { inner: boxed_instance.clone(), create_index_svc, @@ -990,6 +1131,7 @@ impl ControlPlaneServiceTowerLayerStack { toggle_source_svc, delete_source_svc, get_or_create_open_shards_svc, + get_debug_state_svc, }; ControlPlaneServiceClient::new(tower_svc_stack) } @@ -1119,6 +1261,15 @@ where GetOrCreateOpenShardsResponse, crate::control_plane::ControlPlaneError, >, + > + + tower::Service< + GetDebugStateRequest, + Response = GetDebugStateResponse, + Error = crate::control_plane::ControlPlaneError, + Future = BoxFuture< + GetDebugStateResponse, + crate::control_plane::ControlPlaneError, + >, >, { async fn create_index( @@ -1159,6 +1310,12 @@ where ) -> crate::control_plane::ControlPlaneResult { self.call(request).await } + async fn get_debug_state( + &mut self, + request: GetDebugStateRequest, + ) -> crate::control_plane::ControlPlaneResult { + self.call(request).await + } } #[derive(Debug, Clone)] pub struct ControlPlaneServiceGrpcClientAdapter { @@ -1256,6 +1413,16 @@ where .map(|response| response.into_inner()) .map_err(|error| error.into()) } + async fn get_debug_state( + &mut self, + request: GetDebugStateRequest, + ) -> crate::control_plane::ControlPlaneResult { + self.inner + .get_debug_state(request) + .await + .map(|response| response.into_inner()) + .map_err(|error| error.into()) + } } #[derive(Debug)] pub struct ControlPlaneServiceGrpcServerAdapter { @@ -1338,6 +1505,17 @@ for ControlPlaneServiceGrpcServerAdapter { .map(tonic::Response::new) .map_err(|error| error.into()) } + async fn get_debug_state( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner + .clone() + .get_debug_state(request.into_inner()) + .await + .map(tonic::Response::new) + .map_err(|error| error.into()) + } } /// Generated client implementations. pub mod control_plane_service_grpc_client { @@ -1617,6 +1795,37 @@ pub mod control_plane_service_grpc_client { ); self.inner.unary(req, path, codec).await } + /// Return some innerstate of the control plane meant to assist debugging. + pub async fn get_debug_state( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/quickwit.control_plane.ControlPlaneService/GetDebugState", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "quickwit.control_plane.ControlPlaneService", + "GetDebugState", + ), + ); + self.inner.unary(req, path, codec).await + } } } /// Generated server implementations. @@ -1675,6 +1884,14 @@ pub mod control_plane_service_grpc_server { tonic::Response, tonic::Status, >; + /// Return some innerstate of the control plane meant to assist debugging. + async fn get_debug_state( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; } #[derive(Debug)] pub struct ControlPlaneServiceGrpcServer { @@ -2047,6 +2264,52 @@ pub mod control_plane_service_grpc_server { }; Box::pin(fut) } + "/quickwit.control_plane.ControlPlaneService/GetDebugState" => { + #[allow(non_camel_case_types)] + struct GetDebugStateSvc(pub Arc); + impl< + T: ControlPlaneServiceGrpc, + > tonic::server::UnaryService + for GetDebugStateSvc { + type Response = super::GetDebugStateResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + (*inner).get_debug_state(request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = GetDebugStateSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } _ => { Box::pin(async move { Ok( diff --git a/quickwit/quickwit-serve/src/debugging_api.rs b/quickwit/quickwit-serve/src/debugging_api.rs new file mode 100644 index 00000000000..c6616c37be5 --- /dev/null +++ b/quickwit/quickwit-serve/src/debugging_api.rs @@ -0,0 +1,57 @@ +// Copyright (C) 2023 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use quickwit_proto::control_plane::{ + ControlPlaneService, ControlPlaneServiceClient, GetDebugStateRequest, +}; + +#[derive(utoipa::OpenApi)] +#[openapi(paths(debugging_handler))] +/// Endpoints which are weirdly tied to another crate with no +/// other bits of information attached. +/// +/// If a crate plans to encompass different schemas, handlers, etc... +/// Then it should have it's own specific API group. +pub struct DebugApi; + +#[utoipa::path( + get, + tag = "Get debug information for node", + path = "/", + responses( + (status = 200, description = "Successfully fetched debugging info.", body = GetDebugStateRequestResponse), + ), +)] +/// Get Node debug information. +/// +/// The format is not guaranteed to ever be stable, and is meant to provide some introspection to +/// help with debugging. +pub async fn debugging_handler( + mut control_plane_service_client: ControlPlaneServiceClient, +) -> impl warp::Reply { + let debug_info = control_plane_service_client + .get_debug_state(GetDebugStateRequest {}) + .await; + crate::json_api_response::JsonApiResponse::new( + &debug_info, + // TODO error code on error + hyper::StatusCode::OK, + &crate::format::BodyFormat::PrettyJson, + ) +} diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index be700ffac0f..01dd1e85faa 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -19,6 +19,7 @@ mod build_info; mod cluster_api; +mod debugging_api; mod delete_task_api; mod elastic_search_api; mod format; diff --git a/quickwit/quickwit-serve/src/openapi.rs b/quickwit/quickwit-serve/src/openapi.rs index 9a80754ffc5..24319cfb36e 100644 --- a/quickwit/quickwit-serve/src/openapi.rs +++ b/quickwit/quickwit-serve/src/openapi.rs @@ -28,6 +28,7 @@ use utoipa::openapi::Tag; use utoipa::OpenApi; use crate::cluster_api::ClusterApi; +use crate::debugging_api::DebugApi; use crate::delete_task_api::DeleteTaskApi; use crate::elastic_search_api::ElasticCompatibleApi; use crate::health_check_api::HealthCheckApi; @@ -76,12 +77,14 @@ pub fn build_docs() -> utoipa::openapi::OpenApi { Tag::new("Indexing"), Tag::new("Splits"), Tag::new("Jaeger"), + Tag::new("Debugging"), ]; docs_base.tags = Some(tags); // Routing docs_base.merge_components_and_paths(HealthCheckApi::openapi().with_path_prefix("/health")); docs_base.merge_components_and_paths(MetricsApi::openapi().with_path_prefix("/metrics")); + docs_base.merge_components_and_paths(DebugApi::openapi().with_path_prefix("/debugging")); docs_base.merge_components_and_paths(ClusterApi::openapi().with_path_prefix("/api/v1")); docs_base.merge_components_and_paths(DeleteTaskApi::openapi().with_path_prefix("/api/v1")); docs_base.merge_components_and_paths(IndexApi::openapi().with_path_prefix("/api/v1")); diff --git a/quickwit/quickwit-serve/src/rest.rs b/quickwit/quickwit-serve/src/rest.rs index 8b9fc868e07..2e89fd28180 100644 --- a/quickwit/quickwit-serve/src/rest.rs +++ b/quickwit/quickwit-serve/src/rest.rs @@ -33,6 +33,7 @@ use tracing::{error, info}; use warp::{redirect, Filter, Rejection, Reply}; use crate::cluster_api::cluster_handler; +use crate::debugging_api::debugging_handler; use crate::delete_task_api::delete_task_api_handlers; use crate::elastic_search_api::elastic_api_handlers; use crate::health_check_api::health_check_handlers; @@ -87,6 +88,12 @@ pub(crate) async fn start_rest_server( // `/metrics` route. let metrics_routes = warp::path("metrics").and(warp::get()).map(metrics_handler); + // `/debugging` route. + let control_plane_service = quickwit_services.control_plane_service.clone(); + let debugging_routes = warp::path("debugging") + .and(warp::get()) + .then(move || debugging_handler(control_plane_service.clone())); + // `/api/v1/*` routes. let api_v1_root_route = api_v1_routes(quickwit_services.clone()); @@ -109,6 +116,7 @@ pub(crate) async fn start_rest_server( .or(ui_handler()) .or(health_check_routes) .or(metrics_routes) + .or(debugging_routes) .with(request_counter) .recover(recover_fn) .with(extra_headers)