[reconfigurator] clickhouse-admin SMF service with dropshot server (#…

…6304)

## Overview

New SMF service in `clickhouse` and `clickhouse_keeper` zones which runs
a dropshot server. The API contains a single `/node/address` endpoint to
retrieve the node's listen address. Other endpoints will be added in
future PRs.

## Purpose

This server will be used to manage ClickHouse server and Keeper nodes.
For now it performs a single basic action to keep the size of this PR
small, but this server will perform other actions like generating the
XML config files, retrieving the state of the node etc.

## Testing

I've deployed locally with the following results:

```console
root@oxz_switch:~# curl http://[fd00:1122:3344:101::e]:8888/node/address
{"clickhouse_address":"[fd00:1122:3344:101::e]:8123"}
```

```console
root@oxz_clickhouse_2c213ff2:~# cat /var/svc/log/oxide-clickhouse-admin:default.log
[ Aug 14 06:54:42 Enabled. ]
[ Aug 14 06:54:42 Rereading configuration. ]
[ Aug 14 06:54:45 Rereading configuration. ]
[ Aug 14 06:54:46 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/clickhouse-admin/bin/clickhouse-admin run -c /var/svc/manifest/site/clickhouse-admin/config.toml -a [fd00:1122:3344:101::e]:8123 -H [fd00:1122:3344:101::e]:8888 &"). ]
[ Aug 14 06:54:46 Method "start" exited with status 0. ]
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"clickhouse-admin","level":30,"time":"2024-08-14T06:54:46.721122327Z","hostname":"oxz_clickhouse_2c213ff2-6544-4316-939f-b51749cf3222","pid":5169,"local_addr":"[fd00:1122:3344:101::e]:8888","component":"dropshot","file":"/home/coatlicue/.cargo/git/checkouts/dropshot-a4a923d29dccc492/52d900a/dropshot/src/server.rs:205"}
{"msg":"accepted connection","v":0,"name":"clickhouse-admin","level":30,"time":"2024-08-14T06:56:17.908877036Z","hostname":"oxz_clickhouse_2c213ff2-6544-4316-939f-b51749cf3222","pid":5169,"local_addr":"[fd00:1122:3344:101::e]:8888","component":"dropshot","file":"/home/coatlicue/.cargo/git/checkouts/dropshot-a4a923d29dccc492/52d900a/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::2]:37268"}
{"msg":"request completed","v":0,"name":"clickhouse-admin","level":30,"time":"2024-08-14T06:56:17.91734856Z","hostname":"oxz_clickhouse_2c213ff2-6544-4316-939f-b51749cf3222","pid":5169,"uri":"/node/address","method":"GET","req_id":"62a3d8fc-e37e-42aa-a715-52dbce8aa493","remote_addr":"[fd00:1122:3344:101::2]:37268","local_addr":"[fd00:1122:3344:101::e]:8888","component":"dropshot","file":"/home/coatlicue/.cargo/git/checkouts/dropshot-a4a923d29dccc492/52d900a/dropshot/src/server.rs:914","latency_us":3151,"response_code":"200"}
```


Related: #5999

Cargo.toml

@@ -3,6 +3,8 @@ members = [
     "api_identity",
     "bootstore",
     "certificates",
+    "clickhouse-admin",
+    "clickhouse-admin/api",
     "clients/bootstrap-agent-client",
     "clients/cockroach-admin-client",
     "clients/ddm-admin-client",
@@ -112,6 +114,8 @@ default-members = [
     "api_identity",
     "bootstore",
     "certificates",
+    "clickhouse-admin",
+    "clickhouse-admin/api",
     "clients/bootstrap-agent-client",
     "clients/cockroach-admin-client",
     "clients/ddm-admin-client",
@@ -294,6 +298,7 @@ cfg-if = "1.0"
 chrono = { version = "0.4", features = [ "serde" ] }
 ciborium = "0.2.2"
 clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] }
+clickhouse-admin-api = { path = "clickhouse-admin/api" }
 clickward = { git = "https://github.com/oxidecomputer/clickward", rev = "ceec762e6a87d2a22bf56792a3025e145caa095e" }
 cockroach-admin-api = { path = "cockroach-admin/api" }
 cockroach-admin-client = { path = "clients/cockroach-admin-client" }
@@ -417,6 +422,7 @@ nexus-test-utils = { path = "nexus/test-utils" }
 nexus-types = { path = "nexus/types" }
 num-integer = "0.1.46"
 num = { version = "0.4.3", default-features = false, features = [ "libm" ] }
+omicron-clickhouse-admin = { path = "clickhouse-admin" }
 omicron-certificates = { path = "certificates" }
 omicron-cockroach-admin = { path = "cockroach-admin" }
 omicron-common = { path = "common" }

clickhouse-admin/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "omicron-clickhouse-admin"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+
+[dependencies]
+anyhow.workspace = true
+camino.workspace = true
+chrono.workspace = true
+clap.workspace = true
+clickhouse-admin-api.workspace = true
+dropshot.workspace = true
+http.workspace = true
+illumos-utils.workspace = true
+omicron-common.workspace = true
+omicron-uuid-kinds.workspace = true
+schemars.workspace = true
+slog.workspace = true
+slog-async.workspace = true
+slog-dtrace.workspace = true
+slog-error-chain.workspace = true
+serde.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+tokio-postgres.workspace = true
+toml.workspace = true
+
+omicron-workspace-hack.workspace = true
+
+[dev-dependencies]
+expectorate.workspace = true
+nexus-test-utils.workspace = true
+omicron-test-utils.workspace = true
+openapi-lint.workspace = true
+openapiv3.workspace = true
+serde_json.workspace = true
+subprocess.workspace = true
+url.workspace = true
+
+[lints]
+workspace = true

clickhouse-admin/api/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "clickhouse-admin-api"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+
+[lints]
+workspace = true
+
+[dependencies]
+dropshot.workspace = true
+omicron-common.workspace = true
+omicron-uuid-kinds.workspace = true
+omicron-workspace-hack.workspace = true
+schemars.workspace = true
+serde.workspace = true

clickhouse-admin/api/src/lib.rs

@@ -0,0 +1,28 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use dropshot::{HttpError, HttpResponseOk, RequestContext};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use std::net::SocketAddrV6;
+
+#[dropshot::api_description]
+pub trait ClickhouseAdminApi {
+    type Context;
+
+    /// Retrieve the address the ClickHouse server or keeper node is listening on
+    #[endpoint {
+        method = GET,
+        path = "/node/address",
+    }]
+    async fn clickhouse_address(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<ClickhouseAddress>, HttpError>;
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)]
+#[serde(rename_all = "snake_case")]
+pub struct ClickhouseAddress {
+    pub clickhouse_address: SocketAddrV6,
+}

clickhouse-admin/src/bin/clickhouse-admin.rs

@@ -0,0 +1,68 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Executable program to run the Omicron ClickHouse admin interface
+
+use anyhow::anyhow;
+use camino::Utf8PathBuf;
+use clap::Parser;
+use omicron_clickhouse_admin::{Clickward, Config};
+use omicron_common::cmd::fatal;
+use omicron_common::cmd::CmdError;
+use std::net::{SocketAddr, SocketAddrV6};
+
+#[derive(Debug, Parser)]
+#[clap(
+    name = "clickhouse-admin",
+    about = "Omicron ClickHouse cluster admin server"
+)]
+enum Args {
+    /// Start the ClickHouse admin server
+    Run {
+        // TODO: This address is solely for testing now. We should remove it
+        // once we have more endpoints up and running.
+        /// Socket address for a running clickhouse server or keeper instance
+        #[clap(long, short = 'a', action)]
+        clickhouse_address: SocketAddrV6,
+
+        /// Address on which this server should run
+        #[clap(long, short = 'H', action)]
+        http_address: SocketAddrV6,
+
+        /// Path to the server configuration file
+        #[clap(long, short, action)]
+        config: Utf8PathBuf,
+    },
+}
+
+#[tokio::main]
+async fn main() {
+    if let Err(err) = main_impl().await {
+        fatal(err);
+    }
+}
+
+async fn main_impl() -> Result<(), CmdError> {
+    let args = Args::parse();
+
+    match args {
+        Args::Run { clickhouse_address, http_address, config } => {
+            let mut config = Config::from_file(&config)
+                .map_err(|err| CmdError::Failure(anyhow!(err)))?;
+            config.dropshot.bind_address = SocketAddr::V6(http_address);
+
+            let clickward = Clickward::new(clickhouse_address);
+
+            let server =
+                omicron_clickhouse_admin::start_server(clickward, config)
+                    .await
+                    .map_err(|err| CmdError::Failure(anyhow!(err)))?;
+            server.await.map_err(|err| {
+                CmdError::Failure(anyhow!(
+                    "server failed after starting: {err}"
+                ))
+            })
+        }
+    }
+}

clickhouse-admin/src/clickward.rs

@@ -0,0 +1,51 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use clickhouse_admin_api::ClickhouseAddress;
+use dropshot::HttpError;
+use slog_error_chain::{InlineErrorChain, SlogInlineError};
+use std::io;
+use std::net::SocketAddrV6;
+
+#[derive(Debug, thiserror::Error, SlogInlineError)]
+pub enum ClickwardError {
+    #[error("clickward failure")]
+    Failure {
+        #[source]
+        err: io::Error,
+    },
+}
+
+impl From<ClickwardError> for HttpError {
+    fn from(err: ClickwardError) -> Self {
+        match err {
+            ClickwardError::Failure { .. } => {
+                let message = InlineErrorChain::new(&err).to_string();
+                HttpError {
+                    status_code: http::StatusCode::INTERNAL_SERVER_ERROR,
+                    error_code: Some(String::from("Internal")),
+                    external_message: message.clone(),
+                    internal_message: message,
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Clickward {
+    clickhouse_address: SocketAddrV6,
+}
+
+impl Clickward {
+    pub fn new(clickhouse_address: SocketAddrV6) -> Self {
+        Self { clickhouse_address }
+    }
+
+    pub fn clickhouse_address(
+        &self,
+    ) -> Result<ClickhouseAddress, ClickwardError> {
+        Ok(ClickhouseAddress { clickhouse_address: self.clickhouse_address })
+    }
+}

clickhouse-admin/src/config.rs

@@ -0,0 +1,43 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use camino::Utf8Path;
+use camino::Utf8PathBuf;
+use dropshot::ConfigDropshot;
+use dropshot::ConfigLogging;
+use serde::Deserialize;
+use serde::Serialize;
+use slog_error_chain::SlogInlineError;
+use std::io;
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+pub struct Config {
+    pub dropshot: ConfigDropshot,
+    pub log: ConfigLogging,
+}
+impl Config {
+    /// Load a `Config` from the given TOML file
+    pub fn from_file(path: &Utf8Path) -> Result<Self, LoadError> {
+        let contents = std::fs::read_to_string(path)
+            .map_err(|err| LoadError::Read { path: path.to_owned(), err })?;
+        toml::de::from_str(&contents)
+            .map_err(|err| LoadError::Parse { path: path.to_owned(), err })
+    }
+}
+
+#[derive(Debug, thiserror::Error, SlogInlineError)]
+pub enum LoadError {
+    #[error("failed to read {path}")]
+    Read {
+        path: Utf8PathBuf,
+        #[source]
+        err: io::Error,
+    },
+    #[error("failed to parse {path} as TOML")]
+    Parse {
+        path: Utf8PathBuf,
+        #[source]
+        err: toml::de::Error,
+    },
+}

clickhouse-admin/src/context.rs

@@ -0,0 +1,21 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use crate::Clickward;
+use slog::Logger;
+
+pub struct ServerContext {
+    clickward: Clickward,
+    _log: Logger,
+}
+
+impl ServerContext {
+    pub fn new(clickward: Clickward, _log: Logger) -> Self {
+        Self { clickward, _log }
+    }
+
+    pub fn clickward(&self) -> &Clickward {
+        &self.clickward
+    }
+}