Upgrade clickhouse from v22.8.9.24 to v23.8.7.24 (#5127)

oximeter/db/src/client.rs

@@ -50,6 +50,10 @@ use tokio::fs;
 use tokio::sync::Mutex;
 use uuid::Uuid;
 
+const CLICKHOUSE_DB_MISSING: &'static str = "Database oximeter does not exist";
+const CLICKHOUSE_DB_VERSION_MISSING: &'static str =
+    "Table oximeter.version does not exist";
+
 #[usdt::provider(provider = "clickhouse_client")]
 mod probes {
     fn query__start(_: &usdt::UniqueId, sql: &str) {}
@@ -856,10 +860,10 @@ impl Client {
             })?,
             Err(Error::Database(err))
                 // Case 1: The database has not been created.
-                if err.contains("Database oximeter doesn't exist") ||
+                if err.contains(CLICKHOUSE_DB_MISSING) ||
                 // Case 2: The database has been created, but it's old (exists
                 // prior to the version table).
-                    err.contains("Table oximeter.version doesn't exist") =>
+                    err.contains(CLICKHOUSE_DB_VERSION_MISSING) =>
             {
                 warn!(self.log, "oximeter database does not exist, or is out-of-date");
                 0

oximeter/db/src/configs/keeper_config.xml

@@ -9,7 +9,7 @@
     </logger>
 
     <!-- To allow all use :: -->
-    <listen_host from_env="CH_LISTEN_ADDR"/> 
+    <listen_host from_env="CH_LISTEN_ADDR"/>
     <path replace="true" from_env="CH_DATASTORE"/>
 
     <keeper_server>

smf/clickhouse_keeper/keeper_config.xml

@@ -9,7 +9,7 @@
     </logger>
 
     <!-- To allow all use :: -->
-    <listen_host from_env="CH_LISTEN_ADDR"/> 
+    <listen_host from_env="CH_LISTEN_ADDR"/>
     <path replace="true" from_env="CH_DATASTORE"/>
 
     <keeper_server>

smf/clickhouse_keeper/method_script.sh

@@ -39,20 +39,22 @@ KEEPER_HOST_01="${keepers[0]}"
 KEEPER_HOST_02="${keepers[1]}"
 KEEPER_HOST_03="${keepers[2]}"
 
-# Generate unique reproduceable number IDs by removing letters from KEEPER_IDENTIFIER_*
-# Keeper IDs must be numbers, and they cannot be reused (i.e. when a keeper node is 
-# unrecoverable the ID must be changed to something new). 
-# By trimming the hosts we can make sure all keepers will always be up to date when 
-# a new keeper is spun up. Clickhouse does not allow very large numbers, so we will
-# be reducing to 7 characters. This should be enough entropy given the small amount
-# of keepers we have.
+# Generate unique reproduceable number IDs by removing letters from
+# KEEPER_IDENTIFIER_* Keeper IDs must be numbers, and they cannot be reused
+# (i.e. when a keeper node is unrecoverable the ID must be changed to something
+# new). By trimming the hosts we can make sure all keepers will always be up to
+# date when a new keeper is spun up. Clickhouse does not allow very large
+# numbers, so we will be reducing to 7 characters. This should be enough
+# entropy given the small amount of keepers we have.
 KEEPER_ID_01="$( echo "${KEEPER_HOST_01}" | tr -dc [:digit:] | cut -c1-7)"
 KEEPER_ID_02="$( echo "${KEEPER_HOST_02}" | tr -dc [:digit:] | cut -c1-7)"
 KEEPER_ID_03="$( echo "${KEEPER_HOST_03}" | tr -dc [:digit:] | cut -c1-7)"
 
-# Identify the node type this is as this will influence how the config is constructed
-# TODO(https://github.com/oxidecomputer/omicron/issues/3824): There are probably much better ways to do this service name lookup, but this works
-# for now. The services contain the same IDs as the hostnames.
+# Identify the node type this is as this will influence how the config is
+# constructed
+# TODO(https://github.com/oxidecomputer/omicron/issues/3824): There are
+# probably much better ways to do this service name lookup, but this works for
+# now. The services contain the same IDs as the hostnames.
 KEEPER_SVC="$(zonename | tr -dc [:digit:] | cut -c1-7)"
 if [[ $KEEPER_ID_01 == $KEEPER_SVC ]]
 then
@@ -68,9 +70,9 @@ else
     exit "$SMF_EXIT_ERR_CONFIG"
 fi
 
-# Setting environment variables this way is best practice, but has the downside of
-# obscuring the field values to anyone ssh=ing into the zone.  To mitigate this,
-# we will be saving them to ${DATASTORE}/config_env_vars
+# Setting environment variables this way is best practice, but has the downside
+# of obscuring the field values to anyone ssh=ing into the zone.  To mitigate
+# this, we will be saving them to ${DATASTORE}/config_env_vars
 export CH_LOG="${DATASTORE}/clickhouse-keeper.log"
 export CH_ERROR_LOG="${DATASTORE}/clickhouse-keeper.err.log"
 export CH_LISTEN_ADDR=${LISTEN_ADDR}
@@ -103,7 +105,7 @@ CH_KEEPER_HOST_03="${CH_KEEPER_HOST_03}""
 
 echo $content >> "${DATASTORE}/config_env_vars"
 
-# The clickhouse binary must be run from within the directory that contains it. 
+# The clickhouse binary must be run from within the directory that contains it.
 # Otherwise, it does not automatically detect the configuration files, nor does
 # it append them when necessary
 cd /opt/oxide/clickhouse_keeper/

smf/profile/profile

@@ -12,8 +12,8 @@ case "$HOSTNAME" in
     oxz_crucible*)
 	PATH+=:/opt/oxide/crucible/bin
 	;;
-    oxz_clockhouse*)
-	PATH+=:/opt/oxide/clickhouse
+    oxz_clickhouse*)
+	PATH+=:/opt/oxide/clickhouse:/opt/oxide/clickhouse_keeper
 	;;
     oxz_external_dns*|oxz_internal_dns*)
 	PATH+=:/opt/oxide/dns-server/bin