Revert "Revert "Update nightly build and deploy script for arm artifa…

…cts [skip ci] (#9888)""

This reverts commit 5d31719.

jenkins/deploy.sh

@@ -30,6 +30,7 @@
 #   POM_FILE:       Project pom file to be deployed
 #   OUT_PATH:       The path where jar files are
 #   CUDA_CLASSIFIERS:    Comma separated classifiers, e.g., "cuda11,cuda12"
+#   CLASSIFIERS:    Comma separated classifiers, e.g., "cuda11,cuda12,cuda11-arm64,cuda12-arm64"
 ###
 
 set -ex
@@ -48,6 +49,7 @@ ART_GROUP_ID=$(mvnEval $DIST_PL project.groupId)
 ART_VER=$(mvnEval $DIST_PL project.version)
 DEFAULT_CUDA_CLASSIFIER=$(mvnEval $DIST_PL cuda.version)
 CUDA_CLASSIFIERS=${CUDA_CLASSIFIERS:-"$DEFAULT_CUDA_CLASSIFIER"}
+CLASSIFIERS=${CLASSIFIERS:-"$CUDA_CLASSIFIERS"} # default as CUDA_CLASSIFIERS for compatibility
 
 SQL_PL=${SQL_PL:-"sql-plugin"}
 POM_FILE=${POM_FILE:-"$DIST_PL/target/parallel-world/META-INF/maven/${ART_GROUP_ID}/${ART_ID}/pom.xml"}
@@ -57,9 +59,8 @@ SIGN_TOOL=${SIGN_TOOL:-"gpg"}
 FPATH="$OUT_PATH/$ART_ID-$ART_VER"
 DEPLOY_TYPES=''
 DEPLOY_FILES=''
-IFS=',' read -a CUDA_CLASSIFIERS_ARR <<< "$CUDA_CLASSIFIERS"
-DEPLOY_TYPES=$(echo $CUDA_CLASSIFIERS | sed -e 's;[^,]*;jar;g')
-DEPLOY_FILES=$(echo $CUDA_CLASSIFIERS | sed -e "s;\([^,]*\);${FPATH}-\1.jar;g")
+DEPLOY_TYPES=$(echo $CLASSIFIERS | sed -e 's;[^,]*;jar;g')
+DEPLOY_FILES=$(echo $CLASSIFIERS | sed -e "s;\([^,]*\);${FPATH}-\1.jar;g")
 
 # dist does not have javadoc and sources jars, use 'sql-plugin' instead
 source jenkins/version-def.sh >/dev/null 2&>1
@@ -103,4 +104,4 @@ $DEPLOY_CMD -DpomFile=$POM_FILE \
             -Djavadoc=$FPATH-javadoc.jar \
             -Dfiles=$DEPLOY_FILES \
             -Dtypes=$DEPLOY_TYPES \
-            -Dclassifiers=$CUDA_CLASSIFIERS
+            -Dclassifiers=$CLASSIFIERS

jenkins/spark-nightly-build.sh

@@ -42,7 +42,8 @@ ART_GROUP_ID=$(mvnEval project.groupId)
 ART_VER=$(mvnEval project.version)
 DEFAULT_CUDA_CLASSIFIER=${DEFAULT_CUDA_CLASSIFIER:-$(mvnEval cuda.version)} # default cuda version
 CUDA_CLASSIFIERS=${CUDA_CLASSIFIERS:-"$DEFAULT_CUDA_CLASSIFIER"} # e.g. cuda11,cuda12
-IFS=',' read -a CUDA_CLASSIFIERS_ARR <<< "$CUDA_CLASSIFIERS"
+CLASSIFIERS=${CLASSIFIERS:-"$CUDA_CLASSIFIERS"}  # default as CUDA_CLASSIFIERS for compatibility
+IFS=',' read -a CLASSIFIERS_ARR <<< "$CLASSIFIERS"
 TMP_PATH="/tmp/$(date '+%Y-%m-%d')-$$"
 
 DIST_FPATH="$DIST_PL/target/$ART_ID-$ART_VER-$DEFAULT_CUDA_CLASSIFIER"
@@ -72,7 +73,7 @@ function distWithReducedPom {
 
         deploy)
             mvnCmd="deploy:deploy-file"
-            if (( ${#CUDA_CLASSIFIERS_ARR[@]} > 1 )); then
+            if (( ${#CLASSIFIERS_ARR[@]} > 1 )); then
               # try move tmp artifacts back to target folder for simplifying separate release process
               mv ${TMP_PATH}/${ART_ID}-${ART_VER}-*.jar ${DIST_PL}/target/
             fi
@@ -102,6 +103,11 @@ function distWithReducedPom {
 
 # option to skip unit tests. Used in our CI to separate test runs in parallel stages
 SKIP_TESTS=${SKIP_TESTS:-"false"}
+if [[ "${SKIP_TESTS}" == "true" ]]; then
+  # if skip test, we could try speed up build with multiple-threads
+  MVN="${MVN} -T1C"
+fi
+
 set +H # turn off history expansion
 DEPLOY_SUBMODULES=${DEPLOY_SUBMODULES:-"!${DIST_PL}"} # TODO: deploy only required submodules to save time
 for buildver in "${SPARK_SHIM_VERSIONS[@]:1}"; do
@@ -129,25 +135,34 @@ for buildver in "${SPARK_SHIM_VERSIONS[@]:1}"; do
 done
 
 installDistArtifact() {
-  local cuda_classifier="$1"
+  local cuda_version="$1"
+  local opt="$2"
   $MVN -B clean install \
+      $opt \
       $DIST_PROFILE_OPT \
       -Dbuildver=$SPARK_BASE_SHIM_VERSION \
       $MVN_URM_MIRROR \
       -Dmaven.repo.local=$M2DIR \
-      -Dcuda.version=$cuda_classifier \
+      -Dcuda.version=$cuda_version \
       -DskipTests=$SKIP_TESTS
 }
 
 # build extra cuda classifiers
-if (( ${#CUDA_CLASSIFIERS_ARR[@]} > 1 )); then
+if (( ${#CLASSIFIERS_ARR[@]} > 1 )); then
   mkdir -p ${TMP_PATH}
-  for classifier in "${CUDA_CLASSIFIERS_ARR[@]}"; do
+  for classifier in "${CLASSIFIERS_ARR[@]}"; do
     if [ "${classifier}" == "${DEFAULT_CUDA_CLASSIFIER}" ]; then
       echo "skip default: ${DEFAULT_CUDA_CLASSIFIER} in build extra cuda classifiers step..."
       continue
     fi
-    installDistArtifact ${classifier}
+
+    opt=""
+    if [[ "${classifier}" == *"-arm64" ]]; then
+      opt="-Parm64"
+    fi
+    # pass cuda version and extra opt
+    installDistArtifact ${classifier%%-*} ${opt}
+
     # move artifacts to temp for deployment later
     artifactFile="${ART_ID}-${ART_VER}-${classifier}.jar"
     mv ${DIST_PL}/target/${artifactFile} ${TMP_PATH}/