From adfef12c46d8fe966fca225bb0c3d424702e447c Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Mon, 15 Jan 2024 22:09:30 +0800 Subject: [PATCH 1/5] Deploy release candidates to local maven repo for dependency check To fix : https://github.com/NVIDIA/spark-rapids/issues/10164 Deploy release candidates to local maven repo leveraging the release deploy script. Instead of using internal snapshots maven repo(having all the intermediate artifacts), we check dependencies for the release candidates using the local maven repo, which only contains release candidates. Signed-off-by: Tim Liu --- jenkins/deploy.sh | 26 ++++++++++++++++++++++++-- jenkins/spark-nightly-build.sh | 2 +- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/jenkins/deploy.sh b/jenkins/deploy.sh index 16428e121dc..1f9d27b5b72 100755 --- a/jenkins/deploy.sh +++ b/jenkins/deploy.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -50,6 +50,12 @@ ART_VER=$(mvnEval $DIST_PL project.version) DEFAULT_CUDA_CLASSIFIER=$(mvnEval $DIST_PL cuda.version) CUDA_CLASSIFIERS=${CUDA_CLASSIFIERS:-"$DEFAULT_CUDA_CLASSIFIER"} CLASSIFIERS=${CLASSIFIERS:-"$CUDA_CLASSIFIERS"} # default as CUDA_CLASSIFIERS for compatibility +SERVER_ID=${SERVER_ID:-"snapshots"} +SERVER_URL=${SERVER_URL:-"file:/tmp/local-release-repo"} +# Save to be deployed artifact list into the file, e.g. +ARTIFACT_FILE=${ARTIFACT_FILE:-"/tmp/artifact-file"} +# Clean rtifact list file befor saving +rm -rf $ARTIFACT_FILE SQL_PL=${SQL_PL:-"sql-plugin"} POM_FILE=${POM_FILE:-"$DIST_PL/target/parallel-world/META-INF/maven/${ART_GROUP_ID}/${ART_ID}/pom.xml"} @@ -63,7 +69,7 @@ DEPLOY_TYPES=$(echo $CLASSIFIERS | sed -e 's;[^,]*;jar;g') DEPLOY_FILES=$(echo $CLASSIFIERS | sed -e "s;\([^,]*\);${FPATH}-\1.jar;g") # dist does not have javadoc and sources jars, use 'sql-plugin' instead -source jenkins/version-def.sh >/dev/null 2&>1 +source jenkins/version-def.sh >/dev/null 2>&1 echo $SPARK_BASE_SHIM_VERSION SQL_ART_ID=$(mvnEval $SQL_PL project.artifactId) SQL_ART_VER=$(mvnEval $SQL_PL project.version) @@ -96,6 +102,14 @@ echo "Deploy CMD: $DEPLOY_CMD" ###### Deploy the parent pom file ###### $DEPLOY_CMD -Dfile=./pom.xml -DpomFile=./pom.xml +PARENT_ART_ID=$(mvnEval "./" project.artifactId) +echo "$ART_GROUP_ID:$PARENT_ART_ID:$ART_VER:pom" >> $ARTIFACT_FILE + +###### Deploy the jdk-profile pom file ###### +JDK_PROFILES=${JDK_PROFILES:-"jdk-profiles"} +$DEPLOY_CMD -Dfile=$JDK_PROFILES/pom.xml -DpomFile=$JDK_PROFILES/pom.xml +JDK_PROFILES_ART_ID=$(mvnEval "$JDK_PROFILES" project.artifactId) +echo "$ART_GROUP_ID:$JDK_PROFILES_ART_ID:$ART_VER:pom" >> $ARTIFACT_FILE ###### Deploy the artifact jar(s) ###### $DEPLOY_CMD -DpomFile=$POM_FILE \ @@ -105,3 +119,11 @@ $DEPLOY_CMD -DpomFile=$POM_FILE \ -Dfiles=$DEPLOY_FILES \ -Dtypes=$DEPLOY_TYPES \ -Dclassifiers=$CLASSIFIERS + +echo "$ART_GROUP_ID:$ART_ID:$ART_VER:pom" >> $ARTIFACT_FILE +echo "$ART_GROUP_ID:$ART_ID:$ART_VER:jar" >> $ARTIFACT_FILE +CLASSLIST="$CLASSIFIERS,sources,javadoc" +CLASSLIST=(${CLASSLIST//','/' '}) +for class in ${CLASSLIST[@]}; do + echo "$ART_GROUP_ID:$ART_ID:$ART_VER:jar:$class" >> $ARTIFACT_FILE +done diff --git a/jenkins/spark-nightly-build.sh b/jenkins/spark-nightly-build.sh index b038cdc1c08..c5ef53da47d 100755 --- a/jenkins/spark-nightly-build.sh +++ b/jenkins/spark-nightly-build.sh @@ -30,7 +30,7 @@ WORKSPACE=${WORKSPACE:-$(pwd)} export M2DIR=${M2DIR:-"$WORKSPACE/.m2"} ## MVN_OPT : maven options environment, e.g. MVN_OPT='-Dspark-rapids-jni.version=xxx' to specify spark-rapids-jni dependency's version. -MVN="mvn -Dmaven.wagon.http.retryHandler.count=3 -DretryFailedDeploymentCount=3 ${MVN_OPT}" +MVN="mvn -Dmaven.wagon.http.retryHandler.count=3 -DretryFailedDeploymentCount=3 ${MVN_OPT} -Psource-javadoc" DIST_PL="dist" function mvnEval { From 4442749cae7b0b40eb543696af53518a6b0abe1b Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Tue, 16 Jan 2024 22:38:24 +0800 Subject: [PATCH 2/5] Add the dependency checking script Signed-off-by: Tim Liu --- jenkins/dependency-check.sh | 43 +++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100755 jenkins/dependency-check.sh diff --git a/jenkins/dependency-check.sh b/jenkins/dependency-check.sh new file mode 100755 index 00000000000..deb12b41ae1 --- /dev/null +++ b/jenkins/dependency-check.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Argument(s): +# ARTIFACT_FILE : Artifact(groupId:artifactId:version:[[packaging]:classifier]) list file +# +# Used environment(s): +# SERVER_ID: The repository id for this deployment. +# SERVER_URL: The url where to deploy artifacts. +# M2_CACHE: Maven local repo +### + +set -ex + +ARTIFACT_FILE=${1:-"/tmp/artifacts-list"} +SERVER_ID=${SERVER_ID:-"snapshots"} +SERVER_URL=${SERVER_URL:-"file:/tmp/local-release-repo"} +M2_CACHE=${M2_CACHE:-"/tmp/m2-cache"} + +remote_maven_repo=$SERVER_ID::default::$SERVER_URL +# Get the spark-rapids-jni and spark-rapids-private jars from OSS Snapshot maven repo +if [ "$SERVER_ID" == "snapshots" ]; then + oss_snapshot_url="https://oss.sonatype.org/content/repositories/snapshots" + remote_maven_repo="$remote_maven_repo,$SERVER_ID::default::$oss_snapshot_url" +fi +while read line; do + artifact=$line # artifact=groupId:artifactId:version:[[packaging]:classifier] + mvn dependency:get -DremoteRepositories=$remote_maven_repo -Dmaven.repo.local=$M2_CACHE -Dartifact=$artifact +done < $ARTIFACT_FILE From 9ae974fb830b080aececc244a35edead294fb8aa Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Mon, 22 Jan 2024 09:50:33 +0800 Subject: [PATCH 3/5] Remove pom files from the dependency file list Signed-off-by: Tim Liu --- jenkins/deploy.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/jenkins/deploy.sh b/jenkins/deploy.sh index 1f9d27b5b72..92b55fafe8e 100755 --- a/jenkins/deploy.sh +++ b/jenkins/deploy.sh @@ -102,14 +102,10 @@ echo "Deploy CMD: $DEPLOY_CMD" ###### Deploy the parent pom file ###### $DEPLOY_CMD -Dfile=./pom.xml -DpomFile=./pom.xml -PARENT_ART_ID=$(mvnEval "./" project.artifactId) -echo "$ART_GROUP_ID:$PARENT_ART_ID:$ART_VER:pom" >> $ARTIFACT_FILE ###### Deploy the jdk-profile pom file ###### JDK_PROFILES=${JDK_PROFILES:-"jdk-profiles"} $DEPLOY_CMD -Dfile=$JDK_PROFILES/pom.xml -DpomFile=$JDK_PROFILES/pom.xml -JDK_PROFILES_ART_ID=$(mvnEval "$JDK_PROFILES" project.artifactId) -echo "$ART_GROUP_ID:$JDK_PROFILES_ART_ID:$ART_VER:pom" >> $ARTIFACT_FILE ###### Deploy the artifact jar(s) ###### $DEPLOY_CMD -DpomFile=$POM_FILE \ From dd327896452e772900d61a650986b2a2a504b4bc Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Mon, 22 Jan 2024 10:04:56 +0800 Subject: [PATCH 4/5] Remove another pom in the dependency check list Signed-off-by: Tim Liu --- jenkins/deploy.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins/deploy.sh b/jenkins/deploy.sh index 92b55fafe8e..15bca120cb0 100755 --- a/jenkins/deploy.sh +++ b/jenkins/deploy.sh @@ -116,7 +116,6 @@ $DEPLOY_CMD -DpomFile=$POM_FILE \ -Dtypes=$DEPLOY_TYPES \ -Dclassifiers=$CLASSIFIERS -echo "$ART_GROUP_ID:$ART_ID:$ART_VER:pom" >> $ARTIFACT_FILE echo "$ART_GROUP_ID:$ART_ID:$ART_VER:jar" >> $ARTIFACT_FILE CLASSLIST="$CLASSIFIERS,sources,javadoc" CLASSLIST=(${CLASSLIST//','/' '}) From 94f3078826e93e76ddff2845b86495505e5a05a9 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Mon, 22 Jan 2024 23:46:32 +0800 Subject: [PATCH 5/5] description what jenkions/dependency-check.sh does --- jenkins/dependency-check.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jenkins/dependency-check.sh b/jenkins/dependency-check.sh index deb12b41ae1..4239c40c664 100755 --- a/jenkins/dependency-check.sh +++ b/jenkins/dependency-check.sh @@ -14,6 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# This file checks whether all the dependency jar or pom files for the specified +# artifacts defined in the file "$ARTIFACT_FILE" are available +# in the "$SERVER_ID::default::$SERVER_URL" maven repo + # Argument(s): # ARTIFACT_FILE : Artifact(groupId:artifactId:version:[[packaging]:classifier]) list file