Reduce the log noise caused by core report summary (#1426)

Signed-off-by: Ahmed Hussein (amahussein) <[email protected]> Contributes to #1416 This change set the log level of ToolTextFileWriter to debug and summarize the logging into a single message to show the final directory
NVIDIA · Nov 19, 2024 · 9cba927 · 9cba927
1 parent 6af4bdb
commit 9cba927
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 8 deletions.
diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/ToolTextFileWriter.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/ToolTextFileWriter.scala
@@ -70,7 +70,7 @@ class ToolTextFileWriter(
     // No need to close the outputStream.
     // Java should handle nested streams automatically.
     utf8Writer.foreach { writer =>
-      logInfo(s"$finalLocationText output location: $textOutputLoc")
+      logDebug(s"$finalLocationText output location: $textOutputLoc")
       writer.flush()
       writer.close()
     }

diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/Profiler.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/Profiler.scala
@@ -119,6 +119,7 @@ class Profiler(hadoopConf: Configuration, appArgs: ProfileArgs, enablePB: Boolea
     progressBar.foreach(_.finishAll())
 
     // Write status reports for all event logs to a CSV file
+    logOutputPath()
     val reportResults = generateStatusResults(appStatusReporter.asScala.values.toSeq)
     ProfileOutputWriter.writeCSVTable("Profiling Status", reportResults, outputDir)
   }

diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/qualification/Qualification.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/qualification/Qualification.scala
@@ -237,13 +237,6 @@ class Qualification(outputPath: String, numRows: Int, hadoopConf: Configuration,
     }
   }
 
-  /**
-   * The outputPath of the current instance of the provider
-   */
-  def getReportOutputPath: String = {
-    s"$outputDir/rapids_4_spark_qualification_output"
-  }
-
   /**
    * Generates a qualification report based on the provided summary information.
    */
@@ -263,6 +256,7 @@ class Qualification(outputPath: String, numRows: Int, hadoopConf: Configuration,
     qWriter.writeStageReport(allAppsSum, order)
     qWriter.writeUnsupportedOpsSummaryCSVReport(allAppsSum)
     val appStatusResult = generateStatusResults(appStatusReporter.asScala.values.toSeq)
+    logOutputPath()
     qWriter.writeStatusReport(appStatusResult, order)
     if (mlOpsEnabled) {
       if (allAppsSum.exists(x => x.mlFunctions.nonEmpty)) {

diff --git a/core/src/main/scala/org/apache/spark/sql/rapids/tool/util/RuntimeReporter.scala b/core/src/main/scala/org/apache/spark/sql/rapids/tool/util/RuntimeReporter.scala
@@ -27,6 +27,9 @@ trait RuntimeReporter extends Logging {
   def generateRuntimeReport(hadoopConf: Option[Configuration] = None): Unit = {
     RuntimeUtil.generateReport(outputDir, hadoopConf)
   }
+  def logOutputPath(): Unit = {
+    logInfo(s"Tools output directory: $outputDir")
+  }
 
   /**
    * Updates the status of "SUCCESS" applications to "SKIPPED" if newer attempts with