From b99a9d6b42505ee0ceb21edad1c81d5d64af5a38 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Fri, 26 Jan 2024 17:17:28 +0800 Subject: [PATCH 1/2] Run '--packages' only with default cuda11 jar As '--packages' only works on the default cuda11 jar, it does not support classifier parameter, refer to issue: https://issues.apache.org/jira/browse/SPARK-20075 We can not specify classifier jar to run plugin tests with '--packages', see below error log: Exception in thread "main" java.lang.IllegalArgumentException: requirement failed: Provided Maven Coordinates must be in the form 'groupId:artifactId:version'. The coordinate provided is: com.nvidia:rapids-4-spark_2.12:23.12.0:jar:cuda12 Signed-off-by: Tim Liu --- jenkins/spark-tests.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh index 0a455afcb10..7a0ea566900 100755 --- a/jenkins/spark-tests.sh +++ b/jenkins/spark-tests.sh @@ -304,10 +304,14 @@ if [[ $TEST_MODE == "DEFAULT" ]]; then PYSP_TEST_spark_shuffle_manager=com.nvidia.spark.rapids.${SHUFFLE_SPARK_SHIM}.RapidsShuffleManager \ ./run_pyspark_from_build.sh - SPARK_SHELL_SMOKE_TEST=1 \ - PYSP_TEST_spark_jars_packages=com.nvidia:rapids-4-spark_${SCALA_BINARY_VER}:${PROJECT_VER} \ - PYSP_TEST_spark_jars_repositories=${PROJECT_REPO} \ - ./run_pyspark_from_build.sh + # As '--packages' only works on the default cuda11 jar, it does not support classifiers + # refer to issue : https://issues.apache.org/jira/browse/SPARK-20075 + if [[ "$CLASSIFIER" == "" || "$CLASSIFIER" == "cuda11" ]]; then + SPARK_SHELL_SMOKE_TEST=1 \ + PYSP_TEST_spark_jars_packages=com.nvidia:rapids-4-spark_${SCALA_BINARY_VER}:${PROJECT_VER} \ + PYSP_TEST_spark_jars_repositories=${PROJECT_REPO} \ + ./run_pyspark_from_build.sh + if # ParquetCachedBatchSerializer cache_test PYSP_TEST_spark_sql_cache_serializer=com.nvidia.spark.ParquetCachedBatchSerializer \ From 677c4fe8738caa92d338852c8270513d7b2c83f3 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Sat, 27 Jan 2024 22:16:25 +0800 Subject: [PATCH 2/2] Add description why need run packages test with ==cuda11 case --- jenkins/spark-tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh index 7a0ea566900..8c7db3479eb 100755 --- a/jenkins/spark-tests.sh +++ b/jenkins/spark-tests.sh @@ -306,6 +306,9 @@ if [[ $TEST_MODE == "DEFAULT" ]]; then # As '--packages' only works on the default cuda11 jar, it does not support classifiers # refer to issue : https://issues.apache.org/jira/browse/SPARK-20075 + # "$CLASSIFIER" == ''" is usally for the case running by developers, + # while "$CLASSIFIER" == "cuda11" is for the case running on CI. + # We expect to run packages test for both cases if [[ "$CLASSIFIER" == "" || "$CLASSIFIER" == "cuda11" ]]; then SPARK_SHELL_SMOKE_TEST=1 \ PYSP_TEST_spark_jars_packages=com.nvidia:rapids-4-spark_${SCALA_BINARY_VER}:${PROJECT_VER} \