From 5b66a15d2947365030195f409fd232a69990d417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karen=20C=C3=A1rcamo?= Date: Thu, 7 Nov 2024 15:09:29 +1300 Subject: [PATCH] [clickhouse] Use chrono::UTC for distributed_ddl_queue endpoint (#7001) Modified the SQL query to set the date/time format to one we can parse. ```console $ curl http://[::1]:8888/distributed-ddl-queue | jq % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 100 1993 100 1993 0 0 4203 0 --:--:-- --:--:-- --:--:-- 4213 [ { "entry": "query-0000000001", "entry_version": 5, "initiator_host": "ixchel", "initiator_port": 22001, "cluster": "oximeter_cluster", "query": "CREATE DATABASE IF NOT EXISTS db1 UUID '701a3dd3-10f0-4f5d-b5b2-0ad11bcf2b17' ON CLUSTER oximeter_cluster", "settings": { "load_balancing": "random" }, "query_create_time": "2024-11-01T03:17:08Z", "host": "::1", "port": 22001, "status": "Finished", "exception_code": 0, "exception_text": "", "query_finish_time": "2024-11-01T03:17:08Z", "query_duration_ms": 3 }, ``` --- Cargo.lock | 1 + clickhouse-admin/src/clickhouse_cli.rs | 5 ++++- clickhouse-admin/types/Cargo.toml | 1 + clickhouse-admin/types/src/lib.rs | 28 ++++++++++++++------------ openapi/clickhouse-admin-server.json | 10 ++++++--- 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77f6a20491..03a4c9fc0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1355,6 +1355,7 @@ dependencies = [ "atomicwrites", "camino", "camino-tempfile", + "chrono", "derive_more", "expectorate", "itertools 0.13.0", diff --git a/clickhouse-admin/src/clickhouse_cli.rs b/clickhouse-admin/src/clickhouse_cli.rs index 440a8f7d9e..fbdbe46e5f 100644 --- a/clickhouse-admin/src/clickhouse_cli.rs +++ b/clickhouse-admin/src/clickhouse_cli.rs @@ -147,7 +147,10 @@ impl ClickhouseCli { self.client_non_interactive( ClickhouseClientType::Server, format!( - "SELECT * FROM system.distributed_ddl_queue WHERE cluster = '{}' FORMAT JSONEachRow", + "SELECT * FROM system.distributed_ddl_queue WHERE cluster = '{}' + SETTINGS date_time_output_format = 'iso', + output_format_json_quote_64bit_integers = '0' + FORMAT JSONEachRow", OXIMETER_CLUSTER ).as_str(), "Retrieve information about distributed ddl queries (ON CLUSTER clause) diff --git a/clickhouse-admin/types/Cargo.toml b/clickhouse-admin/types/Cargo.toml index a7f1da1c86..f57b1c5052 100644 --- a/clickhouse-admin/types/Cargo.toml +++ b/clickhouse-admin/types/Cargo.toml @@ -12,6 +12,7 @@ anyhow.workspace = true atomicwrites.workspace = true camino.workspace = true camino-tempfile.workspace = true +chrono.workspace = true derive_more.workspace = true itertools.workspace = true omicron-common.workspace = true diff --git a/clickhouse-admin/types/src/lib.rs b/clickhouse-admin/types/src/lib.rs index d079cee6ec..e563f6da75 100644 --- a/clickhouse-admin/types/src/lib.rs +++ b/clickhouse-admin/types/src/lib.rs @@ -5,6 +5,7 @@ use anyhow::{bail, Context, Error, Result}; use atomicwrites::AtomicFile; use camino::Utf8PathBuf; +use chrono::{DateTime, Utc}; use derive_more::{Add, AddAssign, Display, From}; use itertools::Itertools; use omicron_common::api::external::Generation; @@ -995,7 +996,7 @@ pub struct DistributedDdlQueue { /// Settings used in the DDL operation pub settings: BTreeMap, /// Query created time - pub query_create_time: String, + pub query_create_time: DateTime, /// Hostname pub host: Ipv6Addr, /// Host Port @@ -1007,9 +1008,9 @@ pub struct DistributedDdlQueue { /// Exception message pub exception_text: String, /// Query finish time - pub query_finish_time: String, + pub query_finish_time: DateTime, /// Duration of query execution (in milliseconds) - pub query_duration_ms: String, + pub query_duration_ms: u64, } impl DistributedDdlQueue { @@ -1036,6 +1037,7 @@ impl DistributedDdlQueue { mod tests { use camino::Utf8PathBuf; use camino_tempfile::Builder; + use chrono::{DateTime, Utc}; use slog::{o, Drain}; use slog_term::{FullFormat, PlainDecorator, TestStdoutWriter}; use std::collections::BTreeMap; @@ -1809,8 +1811,8 @@ snapshot_storage_disk=LocalSnapshotDisk fn test_distributed_ddl_queries_parse_success() { let log = log(); let data = - "{\"entry\":\"query-0000000000\",\"entry_version\":5,\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01 16:16:45\",\"host\":\"::1\",\"port\":22001,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01 16:16:45\",\"query_duration_ms\":\"4\"} -{\"entry\":\"query-0000000000\",\"entry_version\":5,\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01 16:16:45\",\"host\":\"::1\",\"port\":22002,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01 16:16:45\",\"query_duration_ms\":\"4\"} + "{\"entry\":\"query-0000000000\",\"entry_version\":5,\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01T16:16:45Z\",\"host\":\"::1\",\"port\":22001,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01T16:16:45Z\",\"query_duration_ms\":4} +{\"entry\":\"query-0000000000\",\"entry_version\":5,\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01T16:16:45Z\",\"host\":\"::1\",\"port\":22002,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01T16:16:45Z\",\"query_duration_ms\":4} " .as_bytes(); let ddl = DistributedDdlQueue::parse(&log, data).unwrap(); @@ -1826,14 +1828,14 @@ snapshot_storage_disk=LocalSnapshotDisk settings: BTreeMap::from([ ("load_balancing".to_string(), "random".to_string()), ]), - query_create_time: "2024-11-01 16:16:45".to_string(), + query_create_time: "2024-11-01T16:16:45Z".parse::>().unwrap(), host: Ipv6Addr::from_str("::1").unwrap(), port: 22001, exception_code: 0, exception_text: "".to_string(), status: "Finished".to_string(), - query_finish_time: "2024-11-01 16:16:45".to_string(), - query_duration_ms: "4".to_string(), + query_finish_time: "2024-11-01T16:16:45Z".parse::>().unwrap(), + query_duration_ms: 4, }, DistributedDdlQueue{ entry: "query-0000000000".to_string(), @@ -1845,16 +1847,16 @@ snapshot_storage_disk=LocalSnapshotDisk settings: BTreeMap::from([ ("load_balancing".to_string(), "random".to_string()), ]), - query_create_time: "2024-11-01 16:16:45".to_string(), + query_create_time: "2024-11-01T16:16:45Z".parse::>().unwrap(), host: Ipv6Addr::from_str("::1").unwrap(), port: 22002, exception_code: 0, exception_text: "".to_string(), status: "Finished".to_string(), - query_finish_time: "2024-11-01 16:16:45".to_string(), - query_duration_ms: "4".to_string(), + query_finish_time: "2024-11-01T16:16:45Z".parse::>().unwrap(), + query_duration_ms: 4, }, - ]; + ]; assert!(ddl == expected_result); } @@ -1872,7 +1874,7 @@ snapshot_storage_disk=LocalSnapshotDisk fn test_misshapen_distributed_ddl_queries_parse_fail() { let log = log(); let data = - "{\"entry\":\"query-0000000000\",\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01 16:16:45\",\"host\":\"::1\",\"port\":22001,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01 16:16:45\",\"query_duration_ms\":\"4\"} + "{\"entry\":\"query-0000000000\",\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01T16:16:45Z\",\"host\":\"::1\",\"port\":22001,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01T16:16:45Z\",\"query_duration_ms\":4} " .as_bytes(); let result = DistributedDdlQueue::parse(&log, data); diff --git a/openapi/clickhouse-admin-server.json b/openapi/clickhouse-admin-server.json index 52faddb3b9..a15090dd2d 100644 --- a/openapi/clickhouse-admin-server.json +++ b/openapi/clickhouse-admin-server.json @@ -174,15 +174,19 @@ }, "query_create_time": { "description": "Query created time", - "type": "string" + "type": "string", + "format": "date-time" }, "query_duration_ms": { "description": "Duration of query execution (in milliseconds)", - "type": "string" + "type": "integer", + "format": "uint64", + "minimum": 0 }, "query_finish_time": { "description": "Query finish time", - "type": "string" + "type": "string", + "format": "date-time" }, "settings": { "description": "Settings used in the DDL operation",