fix(risedev): always download latest Kafka (#15514)

risingwavelabs · Mar 7, 2024 · cad99ce · cad99ce
1 parent 3a2e574
commit cad99ce
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 12 deletions.
diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
@@ -99,4 +99,17 @@ function filter_stack_trace() {
   | sed -E '/  at ...cargo/d' > tmp
   cp tmp "$1"
   rm tmp
-}
+}
+
+get_latest_kafka_version() {
+    local versions=$(curl -s https://downloads.apache.org/kafka/ | grep -Eo 'href="[0-9]+\.[0-9]+\.[0-9]+/"' | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+")
+    # Sort the version numbers and get the latest one
+    local latest_version=$(echo "$versions" | sort -V | tail -n1)
+    echo $latest_version
+}
+
+get_latest_kafka_download_url() {
+    local latest_version=$(get_latest_kafka_version)
+    local download_url="https://downloads.apache.org/kafka/${latest_version}/kafka_2.13-${latest_version}.tgz"
+    echo $download_url
+}
diff --git a/ci/scripts/gen-flamegraph.sh b/ci/scripts/gen-flamegraph.sh
@@ -97,8 +97,8 @@ install_all() {
   promql --version
 
   echo ">>> Installing Kafka"
-  wget https://archive.apache.org/dist/kafka/3.4.1/kafka_2.13-3.4.1.tgz
-  tar -zxvf kafka_2.13-3.4.1.tgz
+  wget $(get_latest_kafka_download_url) -O kafka_latest.tgz
+  tar -zxvf kafka_latest.tgz
 
   echo ">>> Installing nexmark bench"
   buildkite-agent artifact download nexmark-server /usr/local/bin

diff --git a/src/risedevtool/kafka.toml b/src/risedevtool/kafka.toml
@@ -2,11 +2,6 @@ extend = "common.toml"
 
 [env]
 KAFKA_DOWNLOAD_PATH = "${PREFIX_TMP}/kafka.tgz"
-KAFKA_VERSION = "3.4.1"
-KAFKA_SCALA_VERSION = "2.13"
-KAFKA_RELEASE = "kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}"
-KAFKA_DOWNLOAD_RECENT_TGZ = "https://dlcdn.apache.org/kafka/${KAFKA_VERSION}/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz"
-KAFKA_DOWNLOAD_ARCHIVE_TGZ = "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz"
 
 [tasks.download-kafka]
 private = true
@@ -20,10 +15,18 @@ set -e
 if [ -d "${PREFIX_BIN}/kafka" ]; then
     exit 0
 fi
-echo "Kafka not found, download ${KAFKA_RELEASE}"
-curl -fL -o "${KAFKA_DOWNLOAD_PATH}" "${KAFKA_DOWNLOAD_RECENT_TGZ}" \
- || curl -fL -o "${KAFKA_DOWNLOAD_PATH}" "${KAFKA_DOWNLOAD_ARCHIVE_TGZ}"
+
+get_latest_kafka_version() {
+    local versions=$(curl -s https://downloads.apache.org/kafka/ | grep -Eo 'href="[0-9]+\.[0-9]+\.[0-9]+/"' | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+")
+    # Sort the version numbers and get the latest one
+    local latest_version=$(echo "$versions" | sort -V | tail -n1)
+    echo $latest_version
+}
+
+echo "Kafka not found, downloading..."
+latest_version=$(get_latest_kafka_version)
+curl -fL -o "${KAFKA_DOWNLOAD_PATH}" "https://downloads.apache.org/kafka/${latest_version}/kafka_2.13-${latest_version}.tgz"
 tar -xf "${KAFKA_DOWNLOAD_PATH}" -C "${PREFIX_TMP}"
-mv "${PREFIX_TMP}/${KAFKA_RELEASE}" "${PREFIX_BIN}/kafka"
+mv "${PREFIX_TMP}/kafka_2.13-${latest_version}" "${PREFIX_BIN}/kafka"
 rm ${KAFKA_DOWNLOAD_PATH}
 '''