From cad99cecf4f99186df536a857f5b137ba666f8cf Mon Sep 17 00:00:00 2001
From: Eric Fu <eric@singularity-data.com>
Date: Thu, 7 Mar 2024 18:03:54 +0800
Subject: [PATCH] fix(risedev): always download latest Kafka (#15514)

---
 ci/scripts/common.sh         | 15 ++++++++++++++-
 ci/scripts/gen-flamegraph.sh |  4 ++--
 src/risedevtool/kafka.toml   | 21 ++++++++++++---------
 3 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
index 35601c72e4e62..f579f94c464b3 100755
--- a/ci/scripts/common.sh
+++ b/ci/scripts/common.sh
@@ -99,4 +99,17 @@ function filter_stack_trace() {
   | sed -E '/  at ...cargo/d' > tmp
   cp tmp "$1"
   rm tmp
-}
\ No newline at end of file
+}
+
+get_latest_kafka_version() {
+    local versions=$(curl -s https://downloads.apache.org/kafka/ | grep -Eo 'href="[0-9]+\.[0-9]+\.[0-9]+/"' | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+")
+    # Sort the version numbers and get the latest one
+    local latest_version=$(echo "$versions" | sort -V | tail -n1)
+    echo $latest_version
+}
+
+get_latest_kafka_download_url() {
+    local latest_version=$(get_latest_kafka_version)
+    local download_url="https://downloads.apache.org/kafka/${latest_version}/kafka_2.13-${latest_version}.tgz"
+    echo $download_url
+}
diff --git a/ci/scripts/gen-flamegraph.sh b/ci/scripts/gen-flamegraph.sh
index 62e1543bd0e5b..d5d4a9eed480a 100755
--- a/ci/scripts/gen-flamegraph.sh
+++ b/ci/scripts/gen-flamegraph.sh
@@ -97,8 +97,8 @@ install_all() {
   promql --version
 
   echo ">>> Installing Kafka"
-  wget https://archive.apache.org/dist/kafka/3.4.1/kafka_2.13-3.4.1.tgz
-  tar -zxvf kafka_2.13-3.4.1.tgz
+  wget $(get_latest_kafka_download_url) -O kafka_latest.tgz
+  tar -zxvf kafka_latest.tgz
 
   echo ">>> Installing nexmark bench"
   buildkite-agent artifact download nexmark-server /usr/local/bin
diff --git a/src/risedevtool/kafka.toml b/src/risedevtool/kafka.toml
index 78dc075cd084e..4353e30aef919 100644
--- a/src/risedevtool/kafka.toml
+++ b/src/risedevtool/kafka.toml
@@ -2,11 +2,6 @@ extend = "common.toml"
 
 [env]
 KAFKA_DOWNLOAD_PATH = "${PREFIX_TMP}/kafka.tgz"
-KAFKA_VERSION = "3.4.1"
-KAFKA_SCALA_VERSION = "2.13"
-KAFKA_RELEASE = "kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}"
-KAFKA_DOWNLOAD_RECENT_TGZ = "https://dlcdn.apache.org/kafka/${KAFKA_VERSION}/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz"
-KAFKA_DOWNLOAD_ARCHIVE_TGZ = "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz"
 
 [tasks.download-kafka]
 private = true
@@ -20,10 +15,18 @@ set -e
 if [ -d "${PREFIX_BIN}/kafka" ]; then
     exit 0
 fi
-echo "Kafka not found, download ${KAFKA_RELEASE}"
-curl -fL -o "${KAFKA_DOWNLOAD_PATH}" "${KAFKA_DOWNLOAD_RECENT_TGZ}" \
- || curl -fL -o "${KAFKA_DOWNLOAD_PATH}" "${KAFKA_DOWNLOAD_ARCHIVE_TGZ}"
+
+get_latest_kafka_version() {
+    local versions=$(curl -s https://downloads.apache.org/kafka/ | grep -Eo 'href="[0-9]+\.[0-9]+\.[0-9]+/"' | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+")
+    # Sort the version numbers and get the latest one
+    local latest_version=$(echo "$versions" | sort -V | tail -n1)
+    echo $latest_version
+}
+
+echo "Kafka not found, downloading..."
+latest_version=$(get_latest_kafka_version)
+curl -fL -o "${KAFKA_DOWNLOAD_PATH}" "https://downloads.apache.org/kafka/${latest_version}/kafka_2.13-${latest_version}.tgz"
 tar -xf "${KAFKA_DOWNLOAD_PATH}" -C "${PREFIX_TMP}"
-mv "${PREFIX_TMP}/${KAFKA_RELEASE}" "${PREFIX_BIN}/kafka"
+mv "${PREFIX_TMP}/kafka_2.13-${latest_version}" "${PREFIX_BIN}/kafka"
 rm ${KAFKA_DOWNLOAD_PATH}
 '''