add some more expressions in ignore list, update output format

NVIDIA · Dec 12, 2023 · 9a649c0 · 9a649c0
1 parent 246fe4c
commit 9a649c0
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 12 deletions.
diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/planparser/SQLPlanParser.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/planparser/SQLPlanParser.scala
@@ -78,7 +78,8 @@ object SQLPlanParser extends Logging {
 
   val windowFunctionPattern = """(\w+)\(""".r
 
-  val ignoreExpressions = Array("any", "cast", "decimal", "decimaltype", "every", "some",
+  val ignoreExpressions = Array("any", "cast", "ansi_cast", "decimal", "decimaltype", "every",
+    "some", "merge_max", "merge_min", "merge_sum", "merge_count", "merge_avg", "merge_first",
     "list",
     // current_database does not cause any CPU fallbacks
     "current_database",

diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualOutputWriter.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/qualification/QualOutputWriter.scala
@@ -24,7 +24,7 @@ import com.nvidia.spark.rapids.tool.profiling.ProfileUtils.replaceDelimiter
 import com.nvidia.spark.rapids.tool.qualification.QualOutputWriter.{CLUSTER_ID, CLUSTER_ID_STR_SIZE, JOB_ID, JOB_ID_STR_SIZE, RUN_NAME, RUN_NAME_STR_SIZE, TEXT_DELIMITER}
 import org.apache.hadoop.conf.Configuration
 
-import org.apache.spark.sql.rapids.tool.ToolUtils
+import org.apache.spark.sql.rapids.tool.{IgnoreExecs, ToolUtils}
 import org.apache.spark.sql.rapids.tool.qualification.{EstimatedPerSQLSummaryInfo, EstimatedSummaryInfo, QualificationAppInfo, QualificationSummaryInfo, StatusSummaryInfo}
 import org.apache.spark.sql.rapids.tool.util._
 
@@ -939,19 +939,26 @@ object QualOutputWriter {
     sumInfo.stageInfo.collect {
       case info if info.unsupportedExecs.nonEmpty =>
         val stageAppDuration = info.stageWallclockDuration
-        val unsupportedExecs = info.unsupportedExecs.mkString(";")
-        val data = ListBuffer[(String, Int)](
-          reformatCSVFunc(appId) -> headersAndSizes(APP_ID_STR),
-          reformatCSVFunc(unsupportedExecs) -> headersAndSizes(UNSUPPORTED_TYPE),
-          info.stageId.toString -> headersAndSizes(STAGE_ID_STR),
-          stageAppDuration.toString -> headersAndSizes(STAGE_WALLCLOCK_DUR_STR),
-          appDuration.toString -> headersAndSizes(APP_DUR_STR),
-          recommendation -> headersAndSizes(SPEEDUP_BUCKET_STR)
-        )
-        constructOutputRow(data, delimiter, prettyPrint)
+        val filteredUnsupportedExecs = info.unsupportedExecs.filterNot(
+          x => IgnoreExecs.getAllIgnoreExecs.contains(x))
+        val unsupportedExecsStr = filteredUnsupportedExecs.mkString(";")
+        if (unsupportedExecsStr.nonEmpty) {
+          val data = ListBuffer[(String, Int)](
+            reformatCSVFunc(appId) -> headersAndSizes(APP_ID_STR),
+            reformatCSVFunc(unsupportedExecsStr) -> headersAndSizes(UNSUPPORTED_TYPE),
+            info.stageId.toString -> headersAndSizes(STAGE_ID_STR),
+            stageAppDuration.toString -> headersAndSizes(STAGE_WALLCLOCK_DUR_STR),
+            appDuration.toString -> headersAndSizes(APP_DUR_STR),
+            recommendation -> headersAndSizes(SPEEDUP_BUCKET_STR)
+          )
+          constructOutputRow(data, delimiter, prettyPrint)
+        } else {
+          ""
+        }
     }
   }
 
+
   def constructUnsupportedOperatorsInfo(
       sumInfo: QualificationSummaryInfo,
       headersAndSizes: LinkedHashMap[String, Int],