Fix Spark 3.5.0 shell classloader issue with the plugin

Signed-off-by: Thomas Graves <[email protected]>
NVIDIA · Oct 20, 2023 · f38e136 · f38e136
1 parent 1baa350
commit f38e136
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala
@@ -149,7 +149,10 @@ object ShimLoader extends Logging {
         // fast path
         logInfo(s"findURLClassLoader found a URLClassLoader $urlCl")
         Option(urlCl)
-      case replCl if replCl.getClass.getName == "org.apache.spark.repl.ExecutorClassLoader" =>
+      case replCl if replCl.getClass.getName == "org.apache.spark.repl.ExecutorClassLoader" ||
+          replCl.getClass.getName == "org.apache.spark.executor.ExecutorClassLoader" =>
+        // Spark 3.5.0 changed the package of ExecutorClassLoader so we check for it being
+        // either old package name or new one.
         // https://issues.apache.org/jira/browse/SPARK-18646
         val parentLoader = MethodUtils.invokeMethod(replCl, true, "parentLoader")
           .asInstanceOf[ClassLoader]