Upgrade to Spark 3.4.1 on kubernetes

astrolabsoftware · Nov 20, 2023 · 7e27619 · 7e27619
1 parent 048d87b
commit 7e27619
Showing 1 changed file with 3 additions and 4 deletions.
diff --git a/conf.sh b/conf.sh
@@ -30,17 +30,16 @@ fi
 
 # Spark parameters
 # ----------------
-# Assuming Scala 2.11
 
 # Spark image tag
 # Spark image is built here: https://github.com/astrolabsoftware/k8s-spark-py/
-SPARK_IMAGE_TAG="k8s-3.2.3"
+SPARK_IMAGE_TAG="k8s-3.4.1"
 
 # Spark version
-SPARK_VERSION="3.2.3"
+SPARK_VERSION="3.4.1"
 
 # Name for the Spark archive
-SPARK_NAME="spark-${SPARK_VERSION}-bin-hadoop3.2"
+SPARK_NAME="spark-${SPARK_VERSION}-bin-hadoop3"
 
 # Spark install location
 SPARK_INSTALL_DIR="${HOME}/fink-k8s-tmp"