Upgrade clickhouse from v22.8.9.24 to v23.8.7.24

oximeter/db/src/client.rs

@@ -857,9 +857,11 @@ impl Client {
             Err(Error::Database(err))
                 // Case 1: The database has not been created.
                 if err.contains("Database oximeter doesn't exist") ||
+                    err.contains("Database oximeter does not exist") ||
                 // Case 2: The database has been created, but it's old (exists
                 // prior to the version table).
-                    err.contains("Table oximeter.version doesn't exist") =>
+                    err.contains("Table oximeter.version doesn't exist") ||
+                    err.contains("Table oximeter.version does not exist") =>
             {
                 warn!(self.log, "oximeter database does not exist, or is out-of-date");
                 0

oximeter/db/src/configs/keeper_config.xml

@@ -9,13 +9,14 @@
     </logger>
 
     <!-- To allow all use :: -->
-    <listen_host from_env="CH_LISTEN_ADDR"/> 
+    <listen_host from_env="CH_LISTEN_ADDR"/>
     <path replace="true" from_env="CH_DATASTORE"/>
 
     <keeper_server>
         <tcp_port from_env="CH_LISTEN_PORT"/>
         <server_id from_env="CH_KEEPER_ID_CURRENT"/>
         <log_storage_path from_env="CH_LOG_STORAGE_PATH"/>
+        <storage_path from_env="CH_STORAGE_PATH"/>
         <snapshot_storage_path from_env="CH_SNAPSHOT_STORAGE_PATH"/>
         <coordination_settings>
             <operation_timeout_ms>10000</operation_timeout_ms>

smf/clickhouse_keeper/keeper_config.xml

@@ -16,6 +16,7 @@
         <tcp_port from_env="CH_LISTEN_PORT"/>
         <server_id from_env="CH_KEEPER_ID_CURRENT"/>
         <log_storage_path from_env="CH_LOG_STORAGE_PATH"/>
+        <storage_path from_env="CH_STORAGE_PATH"/>
         <snapshot_storage_path from_env="CH_SNAPSHOT_STORAGE_PATH"/>
         <coordination_settings>
             <operation_timeout_ms>10000</operation_timeout_ms>

smf/clickhouse_keeper/method_script.sh

@@ -39,20 +39,22 @@ KEEPER_HOST_01="${keepers[0]}"
 KEEPER_HOST_02="${keepers[1]}"
 KEEPER_HOST_03="${keepers[2]}"
 
-# Generate unique reproduceable number IDs by removing letters from KEEPER_IDENTIFIER_*
-# Keeper IDs must be numbers, and they cannot be reused (i.e. when a keeper node is 
-# unrecoverable the ID must be changed to something new). 
-# By trimming the hosts we can make sure all keepers will always be up to date when 
-# a new keeper is spun up. Clickhouse does not allow very large numbers, so we will
-# be reducing to 7 characters. This should be enough entropy given the small amount
-# of keepers we have.
+# Generate unique reproduceable number IDs by removing letters from
+# KEEPER_IDENTIFIER_* Keeper IDs must be numbers, and they cannot be reused
+# (i.e. when a keeper node is unrecoverable the ID must be changed to something
+# new). By trimming the hosts we can make sure all keepers will always be up to
+# date when a new keeper is spun up. Clickhouse does not allow very large
+# numbers, so we will be reducing to 7 characters. This should be enough
+# entropy given the small amount of keepers we have.
 KEEPER_ID_01="$( echo "${KEEPER_HOST_01}" | tr -dc [:digit:] | cut -c1-7)"
 KEEPER_ID_02="$( echo "${KEEPER_HOST_02}" | tr -dc [:digit:] | cut -c1-7)"
 KEEPER_ID_03="$( echo "${KEEPER_HOST_03}" | tr -dc [:digit:] | cut -c1-7)"
 
-# Identify the node type this is as this will influence how the config is constructed
-# TODO(https://github.com/oxidecomputer/omicron/issues/3824): There are probably much better ways to do this service name lookup, but this works
-# for now. The services contain the same IDs as the hostnames.
+# Identify the node type this is as this will influence how the config is
+# constructed
+# TODO(https://github.com/oxidecomputer/omicron/issues/3824): There are
+# probably much better ways to do this service name lookup, but this works for
+# now. The services contain the same IDs as the hostnames.
 KEEPER_SVC="$(zonename | tr -dc [:digit:] | cut -c1-7)"
 if [[ $KEEPER_ID_01 == $KEEPER_SVC ]]
 then
@@ -68,16 +70,17 @@ else
     exit "$SMF_EXIT_ERR_CONFIG"
 fi
 
-# Setting environment variables this way is best practice, but has the downside of
-# obscuring the field values to anyone ssh=ing into the zone.  To mitigate this,
-# we will be saving them to ${DATASTORE}/config_env_vars
+# Setting environment variables this way is best practice, but has the downside
+# of obscuring the field values to anyone ssh=ing into the zone.  To mitigate
+# this, we will be saving them to ${DATASTORE}/config_env_vars
 export CH_LOG="${DATASTORE}/clickhouse-keeper.log"
 export CH_ERROR_LOG="${DATASTORE}/clickhouse-keeper.err.log"
 export CH_LISTEN_ADDR=${LISTEN_ADDR}
 export CH_DATASTORE=${DATASTORE}
 export CH_LISTEN_PORT=${LISTEN_PORT}
 export CH_KEEPER_ID_CURRENT=${KEEPER_ID_CURRENT}
 export CH_LOG_STORAGE_PATH="${DATASTORE}/log"
+export CH_STORAGE_PATH="${DATASTORE}/storage"
 export CH_SNAPSHOT_STORAGE_PATH="${DATASTORE}/snapshots"
 export CH_KEEPER_ID_01=${KEEPER_ID_01}
 export CH_KEEPER_ID_02=${KEEPER_ID_02}
@@ -93,6 +96,7 @@ CH_DATASTORE="${CH_DATASTORE}"\n\
 CH_LISTEN_PORT="${CH_LISTEN_PORT}"\n\
 CH_KEEPER_ID_CURRENT="${CH_KEEPER_ID_CURRENT}"\n\
 CH_LOG_STORAGE_PATH="${CH_LOG_STORAGE_PATH}"\n\
+CH_STORAGE_PATH="${CH_STORAGE_PATH}"\n\
 CH_SNAPSHOT_STORAGE_PATH="${CH_SNAPSHOT_STORAGE_PATH}"\n\
 CH_KEEPER_ID_01="${CH_KEEPER_ID_01}"\n\
 CH_KEEPER_ID_02="${CH_KEEPER_ID_02}"\n\
@@ -103,7 +107,7 @@ CH_KEEPER_HOST_03="${CH_KEEPER_HOST_03}""
 
 echo $content >> "${DATASTORE}/config_env_vars"
 
-# The clickhouse binary must be run from within the directory that contains it. 
+# The clickhouse binary must be run from within the directory that contains it.
 # Otherwise, it does not automatically detect the configuration files, nor does
 # it append them when necessary
 cd /opt/oxide/clickhouse_keeper/