Update comment to include an example config

Signed-off-by: Partho Sarthi <[email protected]>
NVIDIA · Jan 6, 2025 · 4b57134 · 4b57134
1 parent 790c8b9
commit 4b57134
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 7 deletions.
diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala
@@ -121,6 +121,7 @@ abstract class Platform(var gpuDevice: Option[GpuDevice],
     val clusterProperties: Option[ClusterProperties]) extends Logging {
   val platformName: String
   val defaultGpuDevice: GpuDevice
+  val sparkVersionLabel: String = "Spark version"
 
   // It's not deal to use vars here but to minimize changes and
   // keep backwards compatibility we put them here for now and hopefully
@@ -172,6 +173,13 @@ abstract class Platform(var gpuDevice: Option[GpuDevice],
     }
   }
 
+  /**
+   * Identify the latest supported Spark and RapidsShuffleManager version for the platform.
+   */
+  lazy val latestSupportedShuffleManagerInfo: (String, String) = {
+    supportedShuffleManagerVersionMap.maxBy(_._1)
+  }
+
   /**
    * Checks if the given runtime is supported by the platform.
    */
@@ -555,6 +563,7 @@ abstract class Platform(var gpuDevice: Option[GpuDevice],
 abstract class DatabricksPlatform(gpuDevice: Option[GpuDevice],
     clusterProperties: Option[ClusterProperties]) extends Platform(gpuDevice, clusterProperties) {
   override val defaultGpuDevice: GpuDevice = T4Gpu
+  override val sparkVersionLabel: String = "Databricks runtime"
   override def isPlatformCSP: Boolean = true
 
   override val supportedRuntimes: Set[SparkRuntime.SparkRuntime] = Set(

diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/tuning/AutoTuner.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/tuning/AutoTuner.scala
@@ -771,7 +771,8 @@ class AutoTuner(
           case Some(smVersion) =>
             Right(autoTunerConfigsProvider.buildShuffleManagerClassName(smVersion))
           case None =>
-            Left(autoTunerConfigsProvider.shuffleManagerCommentForUnsupportedVersion(sparkVersion))
+            Left(autoTunerConfigsProvider.shuffleManagerCommentForUnsupportedVersion(
+              sparkVersion, platform))
         }
       case None =>
         Left(autoTunerConfigsProvider.shuffleManagerCommentForMissingVersion)
@@ -1467,11 +1468,17 @@ trait AutoTunerConfigsProvider extends Logging {
     s"com.nvidia.spark.rapids.spark$smVersion.RapidsShuffleManager"
   }
 
-  def shuffleManagerCommentForUnsupportedVersion(sparkVersion: String): String = {
-    s"Cannot recommend RAPIDS Shuffle Manager for unsupported '$sparkVersion' version.\n" +
-     "  To enable RAPIDS Shuffle Manager, set 'spark.shuffle.manager' to a value\n" +
-     "  from the supported versions. \n" +
-    s"  See supported versions: $shuffleManagerDocUrl."
+  def shuffleManagerCommentForUnsupportedVersion(
+      sparkVersion: String, platform: Platform): String = {
+    val (latestSparkVersion, latestSmVersion)  = platform.latestSupportedShuffleManagerInfo
+    // scalastyle:off line.size.limit
+    s"""
+       |Cannot recommend RAPIDS Shuffle Manager for unsupported ${platform.sparkVersionLabel}: '$sparkVersion'.
+       |To enable RAPIDS Shuffle Manager, use a supported ${platform.sparkVersionLabel} (e.g., '$latestSparkVersion')
+       |and set: '--conf spark.shuffle.manager=com.nvidia.spark.rapids.spark$latestSmVersion.RapidsShuffleManager'.
+       |See supported versions: $shuffleManagerDocUrl.
+       |""".stripMargin.trim.replaceAll("\n", "\n  ")
+    // scalastyle:on line.size.limit
   }
 
   def shuffleManagerCommentForMissingVersion: String = {

diff --git a/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala b/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala
@@ -2255,7 +2255,7 @@ We recommend using nodes/workers with more memory. Need at least 7796MB memory."
         fail(s"Expected error comment but got valid RapidsShuffleManager: $smClassName")
       case Left(comment) =>
         assert(comment == ProfilingAutoTunerConfigsProvider
-          .shuffleManagerCommentForUnsupportedVersion(sparkVersion))
+          .shuffleManagerCommentForUnsupportedVersion(sparkVersion, autoTuner.platform))
     }
   }