Smoke test with '--package' to fetch the plugin jar

To fix NVIDIA#10160 Run SPARK_SHELL_SMOKE_TEST with '--package' to fetch the plugin jar Can downloading the plugin jar from specified artifact repo by '--repositories' This test can fetch plugin jar from internal maven repo, maven central, or Sonatype staging repo. Signed-off-by: Tim Liu <[email protected]>
NvTimLiu · Jan 22, 2024 · 574d221 · 574d221
1 parent 645daa1
commit 574d221
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 1 deletion.
diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2020-2023, NVIDIA CORPORATION.
+# Copyright (c) 2020-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -333,6 +333,11 @@ EOF
                 --driver-class-path "${PYSP_TEST_spark_driver_extraClassPath}"
                 --conf spark.executor.extraClassPath="${PYSP_TEST_spark_driver_extraClassPath}"
             )
+        elif [[ -n "$PYSP_TEST_spark_jars_packages" ]]; then
+            SPARK_SHELL_ARGS_ARR+=(--packages "${PYSP_TEST_spark_jars_packages}")
+            if [[ -n "$PYSP_TEST_spark_jars_repositories" ]]; then
+                SPARK_SHELL_ARGS_ARR+=(--repositories "${PYSP_TEST_spark_jars_repositories}")
+            fi
         else
             SPARK_SHELL_ARGS_ARR+=(--jars "${PYSP_TEST_spark_jars}")
         fi

diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh
@@ -304,6 +304,11 @@ if [[ $TEST_MODE == "DEFAULT" ]]; then
   PYSP_TEST_spark_shuffle_manager=com.nvidia.spark.rapids.${SHUFFLE_SPARK_SHIM}.RapidsShuffleManager \
     ./run_pyspark_from_build.sh
 
+  SPARK_SHELL_SMOKE_TEST=1 \
+  PYSP_TEST_spark_jars_packages=com.nvidia:rapids-4-spark_${SCALA_BINARY_VER}:${PROJECT_VER} \
+  PYSP_TEST_spark_jars_repositories=${PROJECT_REPO} \
+    ./run_pyspark_from_build.sh
+
   # ParquetCachedBatchSerializer cache_test
   PYSP_TEST_spark_sql_cache_serializer=com.nvidia.spark.ParquetCachedBatchSerializer \
     ./run_pyspark_from_build.sh -k cache_test