From 8e03d940882cde92ddd1a3b8ece99ae6b8c631fe Mon Sep 17 00:00:00 2001 From: cindyyuanjiang Date: Mon, 16 Dec 2024 18:33:25 -0800 Subject: [PATCH] refactored executorCpuTime and executorDeserializeCpuTime Signed-off-by: cindyyuanjiang --- .../analysis/AppSparkMetricsAnalyzer.scala | 12 +- .../profiling/ProfileClassWarehouse.scala | 109 +++++++++++++----- .../sql/rapids/tool/store/TaskModel.scala | 18 ++- ...on_db_13_3_job_metrics_agg_expectation.csv | 30 ++--- ...on_db_13_3_sql_metrics_agg_expectation.csv | 2 +- ..._db_13_3_stage_metrics_agg_expectation.csv | 30 ++--- .../rapids_duration_and_cpu_expectation.csv | 12 +- ...in_eventlog_jobmetricsagg2_expectation.csv | 2 +- ...oin_eventlog_jobmetricsagg_expectation.csv | 2 +- ...ventlog_jobmetricsaggmulti_expectation.csv | 4 +- ...in_eventlog_sqlmetricsagg2_expectation.csv | 2 +- ...oin_eventlog_sqlmetricsagg_expectation.csv | 2 +- ...ventlog_sqlmetricsaggmulti_expectation.csv | 4 +- ..._eventlog_stagemetricsagg2_expectation.csv | 6 +- ...n_eventlog_stagemetricsagg_expectation.csv | 6 +- ...ntlog_stagemetricsaggmulti_expectation.csv | 12 +- 16 files changed, 149 insertions(+), 104 deletions(-) diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/analysis/AppSparkMetricsAnalyzer.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/analysis/AppSparkMetricsAnalyzer.scala index e701eb431..77993df99 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/analysis/AppSparkMetricsAnalyzer.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/analysis/AppSparkMetricsAnalyzer.scala @@ -431,14 +431,12 @@ class AppSparkMetricsAnalyzer(app: AppBase) extends AppAnalysisBase(app) { val shuffleWriteValues = photonShuffleWriteTimeAccumInfos.flatMap { accumInfo => accumInfo.stageValuesMap.get(sm.stageInfo.stageId) } - (AppSparkMetricsAnalyzer.maxWithEmptyHandling(peakMemoryValues), - TimeUnit.NANOSECONDS.toMillis(shuffleWriteValues.sum)) + (AppSparkMetricsAnalyzer.maxWithEmptyHandling(peakMemoryValues), shuffleWriteValues.sum) } else { // For non-Photon apps, use the task metrics directly. val peakMemoryValues = tasksInStage.map(_.peakExecutionMemory) val shuffleWriteTime = tasksInStage.map(_.sw_writeTime) - (AppSparkMetricsAnalyzer.maxWithEmptyHandling(peakMemoryValues), - TimeUnit.NANOSECONDS.toMillis(shuffleWriteTime.sum)) + (AppSparkMetricsAnalyzer.maxWithEmptyHandling(peakMemoryValues), shuffleWriteTime.sum) } val (durSum, durMax, durMin, durAvg) = AppSparkMetricsAnalyzer.getDurations(tasksInStage) @@ -451,8 +449,8 @@ class AppSparkMetricsAnalyzer(app: AppBase) extends AppAnalysisBase(app) { durMax, durMin, durAvg, - tasksInStage.map(_.executorCPUTime).sum, - tasksInStage.map(_.executorDeserializeCPUTime).sum, + TimeUnit.NANOSECONDS.toMillis(tasksInStage.map(_.executorCPUTime).sum), + TimeUnit.NANOSECONDS.toMillis(tasksInStage.map(_.executorDeserializeCPUTime).sum), tasksInStage.map(_.executorDeserializeTime).sum, tasksInStage.map(_.executorRunTime).sum, tasksInStage.map(_.input_bytesRead).sum, @@ -473,7 +471,7 @@ class AppSparkMetricsAnalyzer(app: AppBase) extends AppAnalysisBase(app) { tasksInStage.map(_.sr_totalBytesRead).sum, tasksInStage.map(_.sw_bytesWritten).sum, tasksInStage.map(_.sw_recordsWritten).sum, - shuffleWriteTimeSum + TimeUnit.NANOSECONDS.toMillis(shuffleWriteTimeSum) // nanoseconds ) stageLevelSparkMetrics(index).put(sm.stageInfo.stageId, stageRow) } diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileClassWarehouse.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileClassWarehouse.scala index ab5abdd2b..11998b0e8 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileClassWarehouse.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ProfileClassWarehouse.scala @@ -421,8 +421,8 @@ trait BaseJobStageAggTaskMetricsProfileResult extends ProfileResult { def durationMax: Long def durationMin: Long def durationAvg: Double - def executorCPUTimeSum: Long - def executorDeserializeCpuTimeSum: Long + def executorCPUTimeSum: Long // milliseconds + def executorDeserializeCpuTimeSum: Long // milliseconds def executorDeserializeTimeSum: Long def executorRunTimeSum: Long def inputBytesReadSum: Long @@ -443,20 +443,44 @@ trait BaseJobStageAggTaskMetricsProfileResult extends ProfileResult { def srTotalBytesReadSum: Long def swBytesWrittenSum: Long def swRecordsWrittenSum: Long - def swWriteTimeSum: Long + def swWriteTimeSum: Long // milliseconds def idHeader: String - override val outputHeaders = Seq("appIndex", idHeader, "numTasks", "Duration", - "diskBytesSpilled_sum", "duration_sum", "duration_max", "duration_min", - "duration_avg", "executorCPUTime_sum", "executorDeserializeCPUTime_sum", - "executorDeserializeTime_sum", "executorRunTime_sum", "input_bytesRead_sum", - "input_recordsRead_sum", "jvmGCTime_sum", "memoryBytesSpilled_sum", - "output_bytesWritten_sum", "output_recordsWritten_sum", "peakExecutionMemory_max", - "resultSerializationTime_sum", "resultSize_max", "sr_fetchWaitTime_sum", - "sr_localBlocksFetched_sum", "sr_localBytesRead_sum", "sr_remoteBlocksFetched_sum", - "sr_remoteBytesRead_sum", "sr_remoteBytesReadToDisk_sum", "sr_totalBytesRead_sum", - "sw_bytesWritten_sum", "sw_recordsWritten_sum", "sw_writeTime_sum") + override val outputHeaders = { + Seq("appIndex", + idHeader, + "numTasks", + "Duration", + "diskBytesSpilled_sum", + "duration_sum", + "duration_max", + "duration_min", + "duration_avg", + "executorCPUTime_sum", + "executorDeserializeCPUTime_sum", + "executorDeserializeTime_sum", + "executorRunTime_sum", + "input_bytesRead_sum", + "input_recordsRead_sum", + "jvmGCTime_sum", + "memoryBytesSpilled_sum", + "output_bytesWritten_sum", + "output_recordsWritten_sum", + "peakExecutionMemory_max", + "resultSerializationTime_sum", + "resultSize_max", + "sr_fetchWaitTime_sum", + "sr_localBlocksFetched_sum", + "sr_localBytesRead_sum", + "sr_remoteBlocksFetched_sum", + "sr_remoteBytesRead_sum", + "sr_remoteBytesReadToDisk_sum", + "sr_totalBytesRead_sum", + "sw_bytesWritten_sum", + "sw_recordsWritten_sum", + "sw_writeTime_sum") + } val durStr = duration match { case Some(dur) => dur.toString @@ -511,8 +535,8 @@ case class JobAggTaskMetricsProfileResult( durationMax: Long, durationMin: Long, durationAvg: Double, - executorCPUTimeSum: Long, - executorDeserializeCpuTimeSum: Long, + executorCPUTimeSum: Long, // milliseconds + executorDeserializeCpuTimeSum: Long, // milliseconds executorDeserializeTimeSum: Long, executorRunTimeSum: Long, inputBytesReadSum: Long, @@ -533,7 +557,8 @@ case class JobAggTaskMetricsProfileResult( srTotalBytesReadSum: Long, swBytesWrittenSum: Long, swRecordsWrittenSum: Long, - swWriteTimeSum: Long) extends BaseJobStageAggTaskMetricsProfileResult { + swWriteTimeSum: Long // milliseconds + ) extends BaseJobStageAggTaskMetricsProfileResult { override def idHeader = "jobId" } @@ -547,8 +572,8 @@ case class StageAggTaskMetricsProfileResult( durationMax: Long, durationMin: Long, durationAvg: Double, - executorCPUTimeSum: Long, - executorDeserializeCpuTimeSum: Long, + executorCPUTimeSum: Long, // milliseconds + executorDeserializeCpuTimeSum: Long, // milliseconds executorDeserializeTimeSum: Long, executorRunTimeSum: Long, inputBytesReadSum: Long, @@ -569,7 +594,8 @@ case class StageAggTaskMetricsProfileResult( srTotalBytesReadSum: Long, swBytesWrittenSum: Long, swRecordsWrittenSum: Long, - swWriteTimeSum: Long) extends BaseJobStageAggTaskMetricsProfileResult { + swWriteTimeSum: Long // milliseconds + ) extends BaseJobStageAggTaskMetricsProfileResult { override def idHeader = "stageId" } @@ -747,7 +773,7 @@ case class SQLTaskAggMetricsProfileResult( description: String, numTasks: Int, duration: Option[Long], - executorCpuTime: Long, + executorCpuTime: Long, // milliseconds executorRunTime: Long, executorCpuRatio: Double, diskBytesSpilledSum: Long, @@ -755,8 +781,8 @@ case class SQLTaskAggMetricsProfileResult( durationMax: Long, durationMin: Long, durationAvg: Double, - executorCPUTimeSum: Long, - executorDeserializeCpuTimeSum: Long, + executorCPUTimeSum: Long, // milliseconds + executorDeserializeCpuTimeSum: Long, // milliseconds executorDeserializeTimeSum: Long, executorRunTimeSum: Long, inputBytesReadSum: Long, @@ -779,7 +805,8 @@ case class SQLTaskAggMetricsProfileResult( srTotalBytesReadSum: Long, swBytesWrittenSum: Long, swRecordsWrittenSum: Long, - swWriteTimeSum: Long) extends ProfileResult { + swWriteTimeSum: Long // milliseconds + ) extends ProfileResult { override val outputHeaders = Seq("appIndex", "appID", "sqlID", "description", "numTasks", "Duration", "executorCPUTime", "executorRunTime", "executorCPURatio", @@ -924,12 +951,27 @@ case class IOAnalysisProfileResult( } } -case class SQLDurationExecutorTimeProfileResult(appIndex: Int, appId: String, - rootsqlID: Option[Long], sqlID: Long, duration: Option[Long], containsDataset: Boolean, - appDuration: Option[Long], potentialProbs: String, +case class SQLDurationExecutorTimeProfileResult( + appIndex: Int, + appId: String, + rootsqlID: Option[Long], + sqlID: Long, + duration: Option[Long], + containsDataset: Boolean, + appDuration: Option[Long], + potentialProbs: String, executorCpuRatio: Double) extends ProfileResult { - override val outputHeaders = Seq("appIndex", "App ID", "RootSqlID", "sqlID", "SQL Duration", - "Contains Dataset or RDD Op", "App Duration", "Potential Problems", "Executor CPU Time Percent") + override val outputHeaders = { + Seq("appIndex", + "App ID", + "RootSqlID", + "sqlID", + "SQL Duration", + "Contains Dataset or RDD Op", + "App Duration", + "Potential Problems", + "Executor CPU Time Percent") + } val durStr = duration match { case Some(dur) => dur.toString case None => "" @@ -950,8 +992,15 @@ case class SQLDurationExecutorTimeProfileResult(appIndex: Int, appId: String, } override def convertToSeq: Seq[String] = { - Seq(appIndex.toString, rootsqlID.getOrElse("").toString, appId, sqlID.toString, durStr, - containsDataset.toString, appDurStr, potentialStr, execCpuTimePercent) + Seq(appIndex.toString, + rootsqlID.getOrElse("").toString, + appId, + sqlID.toString, + durStr, + containsDataset.toString, + appDurStr, + potentialStr, + execCpuTimePercent) } override def convertToCSVSeq: Seq[String] = { diff --git a/core/src/main/scala/org/apache/spark/sql/rapids/tool/store/TaskModel.scala b/core/src/main/scala/org/apache/spark/sql/rapids/tool/store/TaskModel.scala index 2faf6401f..9ae2f3579 100644 --- a/core/src/main/scala/org/apache/spark/sql/rapids/tool/store/TaskModel.scala +++ b/core/src/main/scala/org/apache/spark/sql/rapids/tool/store/TaskModel.scala @@ -16,8 +16,6 @@ package org.apache.spark.sql.rapids.tool.store -import java.util.concurrent.TimeUnit - import org.apache.spark.TaskFailedReason import org.apache.spark.scheduler.SparkListenerTaskEnd import org.apache.spark.sql.rapids.tool.annotation.Since @@ -40,9 +38,9 @@ case class TaskModel( speculative: Boolean, gettingResultTime: Long, executorDeserializeTime: Long, - executorDeserializeCPUTime: Long, - executorRunTime: Long, - executorCPUTime: Long, + executorDeserializeCPUTime: Long, // nanoseconds + executorRunTime: Long, // milliseconds + executorCPUTime: Long, // nanoseconds peakExecutionMemory: Long, resultSize: Long, jvmGCTime: Long, @@ -59,7 +57,7 @@ case class TaskModel( sr_totalBytesRead: Long, // Note: sw stands for ShuffleWrite sw_bytesWritten: Long, - sw_writeTime: Long, + sw_writeTime: Long, // nanoseconds sw_recordsWritten: Long, input_bytesRead: Long, input_recordsRead: Long, @@ -92,9 +90,9 @@ object TaskModel { event.taskInfo.speculative, event.taskInfo.gettingResultTime, event.taskMetrics.executorDeserializeTime, - TimeUnit.NANOSECONDS.toMillis(event.taskMetrics.executorDeserializeCpuTime), - event.taskMetrics.executorRunTime, - TimeUnit.NANOSECONDS.toMillis(event.taskMetrics.executorCpuTime), + event.taskMetrics.executorDeserializeCpuTime, // nanoseconds + event.taskMetrics.executorRunTime, // milliseconds + event.taskMetrics.executorCpuTime, // nanoseconds event.taskMetrics.peakExecutionMemory, event.taskMetrics.resultSize, event.taskMetrics.jvmGCTime, @@ -109,7 +107,7 @@ object TaskModel { event.taskMetrics.shuffleReadMetrics.localBytesRead, event.taskMetrics.shuffleReadMetrics.totalBytesRead, event.taskMetrics.shuffleWriteMetrics.bytesWritten, - event.taskMetrics.shuffleWriteMetrics.writeTime, + event.taskMetrics.shuffleWriteMetrics.writeTime, // nanoseconds event.taskMetrics.shuffleWriteMetrics.recordsWritten, event.taskMetrics.inputMetrics.bytesRead, event.taskMetrics.inputMetrics.recordsRead, diff --git a/core/src/test/resources/ProfilingExpectations/nds_q88_photon_db_13_3_job_metrics_agg_expectation.csv b/core/src/test/resources/ProfilingExpectations/nds_q88_photon_db_13_3_job_metrics_agg_expectation.csv index 97de9ad08..4df39b334 100644 --- a/core/src/test/resources/ProfilingExpectations/nds_q88_photon_db_13_3_job_metrics_agg_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/nds_q88_photon_db_13_3_job_metrics_agg_expectation.csv @@ -1,16 +1,16 @@ appIndex,jobId,numTasks,Duration,diskBytesSpilled_sum,duration_sum,duration_max,duration_min,duration_avg,executorCPUTime_sum,executorDeserializeCPUTime_sum,executorDeserializeTime_sum,executorRunTime_sum,input_bytesRead_sum,input_recordsRead_sum,jvmGCTime_sum,memoryBytesSpilled_sum,output_bytesWritten_sum,output_recordsWritten_sum,peakExecutionMemory_max,resultSerializationTime_sum,resultSize_max,sr_fetchWaitTime_sum,sr_localBlocksFetched_sum,sr_localBytesRead_sum,sr_remoteBlocksFetched_sum,sr_remoteBytesRead_sum,sr_remoteBytesReadToDisk_sum,sr_totalBytesRead_sum,sw_bytesWritten_sum,sw_recordsWritten_sum,sw_writeTime_sum -1,48,431,237976,0,371230,1032,333,861.3,343288,1261,1286,367785,190131859,8639936081,160,0,0,0,169667947,21,15767,8,3,15657,3,15657,0,31314,18964,431,16 -1,47,432,214657,0,376777,1133,499,872.2,346230,1272,1310,373271,14007280,8639936081,144,0,0,0,159530057,11,15577,4,3,15657,3,15657,0,31314,19008,432,20 -1,46,433,191440,0,457364,3323,391,1056.3,352763,1508,2509,451358,5242628040,8639936081,1912,0,0,0,250840493,9,16203,551,3,15657,3,15657,0,31314,19052,433,16 +1,48,431,237976,0,371230,1032,333,861.3,343507,1365,1286,367785,190131859,8639936081,160,0,0,0,169667947,21,15767,8,3,15657,3,15657,0,31314,18964,431,16 +1,47,432,214657,0,376777,1133,499,872.2,346447,1388,1310,373271,14007280,8639936081,144,0,0,0,159530057,11,15577,4,3,15657,3,15657,0,31314,19008,432,20 +1,46,433,191440,0,457364,3323,391,1056.3,352977,1639,2509,451358,5242628040,8639936081,1912,0,0,0,250840493,9,16203,551,3,15657,3,15657,0,31314,19052,433,16 1,49,1,186241,0,266,266,266,266.0,86,1,1,261,0,0,0,0,0,0,138414192,0,5344,10,209,9196,222,9768,0,18964,44,1,0 -1,45,433,166081,0,415849,1448,339,960.4,349795,1302,1375,412139,2276478144,8639936081,568,0,0,0,195992906,2,15780,7,3,15657,3,15657,0,31314,19052,433,34 -1,44,431,139667,0,398973,1403,365,925.7,354119,1281,1327,395265,1075691986,8639936081,328,0,0,0,188587155,0,15767,10,3,15657,3,15657,0,31314,18964,431,17 +1,45,433,166081,0,415849,1448,339,960.4,350015,1415,1375,412139,2276478144,8639936081,568,0,0,0,195992906,2,15780,7,3,15657,3,15657,0,31314,19052,433,34 +1,44,431,139667,0,398973,1403,365,925.7,354332,1420,1327,395265,1075691986,8639936081,328,0,0,0,188587155,0,15767,10,3,15657,3,15657,0,31314,18964,431,17 1,50,1,122711,0,267,267,267,267.0,71,1,1,262,0,0,0,0,0,0,138414192,0,5343,58,219,9636,213,9372,0,19008,44,1,0 -1,43,432,114755,0,403652,1369,329,934.4,353325,1290,1326,399766,1395949742,8639936081,624,0,0,0,201771890,13,15767,14,3,15657,3,15657,0,31314,19008,432,16 +1,43,432,114755,0,403652,1369,329,934.4,353529,1424,1326,399766,1395949742,8639936081,624,0,0,0,201771890,13,15767,14,3,15657,3,15657,0,31314,19008,432,16 1,51,1,97958,0,386,386,386,386.0,60,1,1,381,0,0,0,0,0,0,138414192,0,5343,154,221,9724,210,9240,0,18964,44,1,0 -1,42,431,89634,0,616500,1899,589,1430.4,378074,1330,1515,612098,16461920726,8639936081,4132,0,0,0,216740322,23,15805,10,3,15657,3,15657,0,31314,18964,431,16 +1,42,431,89634,0,616500,1899,589,1430.4,378287,1521,1515,612098,16461920726,8639936081,4132,0,0,0,216740322,23,15805,10,3,15657,3,15657,0,31314,18964,431,16 1,52,1,71718,0,384,384,384,384.0,54,1,1,379,0,0,0,0,0,0,138414192,0,5343,170,223,9812,210,9240,0,19052,44,1,0 -1,41,431,51085,0,759623,2321,918,1762.5,394996,1460,2027,754015,26337468742,8639936081,7772,0,0,0,250648581,87,16157,170,3,15657,3,15657,0,31314,18964,431,19 +1,41,431,51085,0,759623,2321,918,1762.5,395214,1706,2027,754015,26337468742,8639936081,7772,0,0,0,250648581,87,16157,170,3,15657,3,15657,0,31314,18964,431,19 1,53,1,46297,0,136,136,136,136.0,57,1,1,131,0,0,0,0,0,0,138414192,0,5344,0,214,9416,219,9636,0,19052,44,1,0 1,54,1,23051,0,340,340,340,340.0,36,1,1,334,0,0,0,0,0,0,138414192,0,5343,223,215,9460,217,9548,0,19008,44,1,0 1,31,1,6979,0,6738,6738,6738,6738.0,5104,128,688,6035,349526,86400,53,0,0,0,155563380,1,10759,0,0,0,0,0,0,0,7239,1800,0 @@ -19,20 +19,20 @@ appIndex,jobId,numTasks,Duration,diskBytesSpilled_sum,duration_sum,duration_max, 1,35,1,6925,0,6729,6729,6729,6729.0,157,136,681,6035,12261,1350,53,0,0,0,155199546,1,9839,0,0,0,0,0,0,0,699,165,0 1,38,1,6855,0,6743,6743,6743,6743.0,187,256,688,6035,349526,86400,53,0,0,0,155563380,1,9927,0,0,0,0,0,0,0,7239,1800,0 1,0,1,6033,0,5699,5699,5699,5699.0,422,948,1114,4382,0,0,37,0,0,0,0,8,2794,0,0,0,0,0,0,0,0,0,0 -1,13,200,5707,0,87661,966,349,438.3,9821,427,951,84265,0,0,144,0,0,0,0,9,6258,0,0,0,0,0,0,0,0,0,0 -1,23,200,5479,0,84240,490,355,421.2,5290,200,214,82784,0,0,136,0,0,0,0,0,6214,0,0,0,0,0,0,0,0,0,0 -1,21,200,5271,0,80904,485,353,404.5,6004,203,220,79384,0,0,136,0,0,0,0,1,6302,0,0,0,0,0,0,0,0,0,0 -1,27,200,4728,0,70760,442,309,353.8,4042,200,209,69494,0,0,152,0,0,0,0,10,5788,0,0,0,0,0,0,0,0,0,0 +1,13,200,5707,0,87661,966,349,438.3,9924,528,951,84265,0,0,144,0,0,0,0,9,6258,0,0,0,0,0,0,0,0,0,0 +1,23,200,5479,0,84240,490,355,421.2,5394,292,214,82784,0,0,136,0,0,0,0,0,6214,0,0,0,0,0,0,0,0,0,0 +1,21,200,5271,0,80904,485,353,404.5,6100,304,220,79384,0,0,136,0,0,0,0,1,6302,0,0,0,0,0,0,0,0,0,0 +1,27,200,4728,0,70760,442,309,353.8,4145,287,209,69494,0,0,152,0,0,0,0,10,5788,0,0,0,0,0,0,0,0,0,0 1,3,1,4708,0,4693,4693,4693,4693.0,280,701,804,3796,0,0,26,0,0,0,0,7,2834,0,0,0,0,0,0,0,0,0,0 -1,25,200,4603,0,70379,569,314,351.9,4106,200,216,69040,0,0,168,0,0,0,0,14,5708,0,0,0,0,0,0,0,0,0,0 +1,25,200,4603,0,70379,569,314,351.9,4200,294,216,69040,0,0,168,0,0,0,0,14,5708,0,0,0,0,0,0,0,0,0,0 1,36,1,4556,0,4332,4332,4332,4332.0,3359,95,401,3907,30328,7200,39,0,0,0,155245068,1,10552,0,0,0,0,0,0,0,7719,1920,0 -1,29,200,4555,0,69682,423,310,348.4,3730,200,218,68521,0,0,168,0,0,0,0,9,5748,0,0,0,0,0,0,0,0,0,0 +1,29,200,4555,0,69682,423,310,348.4,3830,272,218,68521,0,0,168,0,0,0,0,9,5748,0,0,0,0,0,0,0,0,0,0 1,32,1,4515,0,4334,4334,4334,4334.0,260,130,404,3907,349526,86400,39,0,0,0,155563380,1,9851,0,0,0,0,0,0,0,7239,1800,0 1,39,1,4488,0,4322,4322,4322,4322.0,112,124,392,3907,349526,86400,39,0,0,0,155563380,1,9926,0,0,0,0,0,0,0,7239,1800,0 1,37,1,4481,0,4334,4334,4334,4334.0,136,144,405,3907,349526,86400,39,0,0,0,155563380,1,9851,0,0,0,0,0,0,0,7239,1800,0 1,40,1,4476,0,4327,4327,4327,4327.0,98,147,394,3907,349526,86400,39,0,0,0,155563380,1,9895,0,0,0,0,0,0,0,7239,1800,0 1,56,1,1055,0,1022,1022,1022,1022.0,758,77,95,901,0,0,0,0,0,0,134218344,6,10091,5,218,9592,220,9680,0,19272,0,0,0 -1,19,200,803,0,11895,145,38,59.5,943,209,252,10017,0,0,56,0,0,0,0,22,2739,0,0,0,0,0,0,0,0,0,0 +1,19,200,803,0,11895,145,38,59.5,1050,321,252,10017,0,0,56,0,0,0,0,22,2739,0,0,0,0,0,0,0,0,0,0 1,26,1,316,0,312,312,312,312.0,2,1,1,306,0,0,0,0,0,0,0,0,3777,0,0,0,0,0,0,0,0,0,0 1,2,1,280,0,267,267,267,267.0,6,4,4,124,0,0,0,0,0,0,0,1,3342,0,0,0,0,0,0,0,0,0,0 1,11,1,264,0,254,254,254,254.0,5,3,3,241,0,0,0,0,0,0,0,0,2913,0,0,0,0,0,0,0,0,0,0 diff --git a/core/src/test/resources/ProfilingExpectations/nds_q88_photon_db_13_3_sql_metrics_agg_expectation.csv b/core/src/test/resources/ProfilingExpectations/nds_q88_photon_db_13_3_sql_metrics_agg_expectation.csv index 63aaf7a1b..5e1116c98 100644 --- a/core/src/test/resources/ProfilingExpectations/nds_q88_photon_db_13_3_sql_metrics_agg_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/nds_q88_photon_db_13_3_sql_metrics_agg_expectation.csv @@ -1,2 +1,2 @@ appIndex,appID,sqlID,description,numTasks,Duration,executorCPUTime,executorRunTime,executorCPURatio,diskBytesSpilled_sum,duration_sum,duration_max,duration_min,duration_avg,executorCPUTime_sum,executorDeserializeCPUTime_sum,executorDeserializeTime_sum,executorRunTime_sum,input_bytesRead_sum,input_recordsRead_sum,jvmGCTime_sum,memoryBytesSpilled_sum,output_bytesWritten_sum,output_recordsWritten_sum,peakExecutionMemory_max,resultSerializationTime_sum,resultSize_max,sr_fetchWaitTime_sum,sr_localBlocksFetched_sum,sr_localBytesRead_sum,sr_remoteBlocksFetched_sum,sr_remoteBytesRead_sum,sr_remoteBytesReadToDisk_sum,sr_totalBytesRead_sum,sw_bytesWritten_sum,sw_recordsWritten_sum,sw_writeTime_sum -1,"app-20240919162642-0000",26,"query88",3472,250542,2883837,3818106,75.53,0,3858136,6743,54,1111.2,2883837,12349,18186,3818106,52997115316,69120188398,16100,0,0,0,250840493,181,16203,1394,1759,201596,1750,201200,0,402796,218614,19946,154 +1,"app-20240919162642-0000",26,"query88",3472,250542,2885555,3818106,75.58,0,3858136,6743,54,1111.2,2885555,13523,18186,3818106,52997115316,69120188398,16100,0,0,0,250840493,181,16203,1394,1759,201596,1750,201200,0,402796,218614,19946,154 diff --git a/core/src/test/resources/ProfilingExpectations/nds_q88_photon_db_13_3_stage_metrics_agg_expectation.csv b/core/src/test/resources/ProfilingExpectations/nds_q88_photon_db_13_3_stage_metrics_agg_expectation.csv index ae34c1ac0..d50fa6860 100644 --- a/core/src/test/resources/ProfilingExpectations/nds_q88_photon_db_13_3_stage_metrics_agg_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/nds_q88_photon_db_13_3_stage_metrics_agg_expectation.csv @@ -1,16 +1,16 @@ appIndex,stageId,numTasks,Duration,diskBytesSpilled_sum,duration_sum,duration_max,duration_min,duration_avg,executorCPUTime_sum,executorDeserializeCPUTime_sum,executorDeserializeTime_sum,executorRunTime_sum,input_bytesRead_sum,input_recordsRead_sum,jvmGCTime_sum,memoryBytesSpilled_sum,output_bytesWritten_sum,output_recordsWritten_sum,peakExecutionMemory_max,resultSerializationTime_sum,resultSize_max,sr_fetchWaitTime_sum,sr_localBlocksFetched_sum,sr_localBytesRead_sum,sr_remoteBlocksFetched_sum,sr_remoteBytesRead_sum,sr_remoteBytesReadToDisk_sum,sr_totalBytesRead_sum,sw_bytesWritten_sum,sw_recordsWritten_sum,sw_writeTime_sum -1,58,431,237799,0,371230,1032,333,861.3,343288,1261,1286,367785,190131859,8639936081,160,0,0,0,169667947,21,15767,8,3,15657,3,15657,0,31314,18964,431,16 -1,54,432,214633,0,376777,1133,499,872.2,346230,1272,1310,373271,14007280,8639936081,144,0,0,0,159530057,11,15577,4,3,15657,3,15657,0,31314,19008,432,20 -1,44,433,191384,0,457364,3323,391,1056.3,352763,1508,2509,451358,5242628040,8639936081,1912,0,0,0,250840493,9,16203,551,3,15657,3,15657,0,31314,19052,433,16 +1,58,431,237799,0,371230,1032,333,861.3,343507,1365,1286,367785,190131859,8639936081,160,0,0,0,169667947,21,15767,8,3,15657,3,15657,0,31314,18964,431,16 +1,54,432,214633,0,376777,1133,499,872.2,346447,1388,1310,373271,14007280,8639936081,144,0,0,0,159530057,11,15577,4,3,15657,3,15657,0,31314,19008,432,20 +1,44,433,191384,0,457364,3323,391,1056.3,352977,1639,2509,451358,5242628040,8639936081,1912,0,0,0,250840493,9,16203,551,3,15657,3,15657,0,31314,19052,433,16 1,61,1,186240,0,266,266,266,266.0,86,1,1,261,0,0,0,0,0,0,138414192,0,5344,10,209,9196,222,9768,0,18964,44,1,0 -1,46,433,166015,0,415849,1448,339,960.4,349795,1302,1375,412139,2276478144,8639936081,568,0,0,0,195992906,2,15780,7,3,15657,3,15657,0,31314,19052,433,34 -1,50,431,139628,0,398973,1403,365,925.7,354119,1281,1327,395265,1075691986,8639936081,328,0,0,0,188587155,0,15767,10,3,15657,3,15657,0,31314,18964,431,17 +1,46,433,166015,0,415849,1448,339,960.4,350015,1415,1375,412139,2276478144,8639936081,568,0,0,0,195992906,2,15780,7,3,15657,3,15657,0,31314,19052,433,34 +1,50,431,139628,0,398973,1403,365,925.7,354332,1420,1327,395265,1075691986,8639936081,328,0,0,0,188587155,0,15767,10,3,15657,3,15657,0,31314,18964,431,17 1,64,1,122708,0,267,267,267,267.0,71,1,1,262,0,0,0,0,0,0,138414192,0,5343,58,219,9636,213,9372,0,19008,44,1,0 -1,48,432,114722,0,403652,1369,329,934.4,353325,1290,1326,399766,1395949742,8639936081,624,0,0,0,201771890,13,15767,14,3,15657,3,15657,0,31314,19008,432,16 +1,48,432,114722,0,403652,1369,329,934.4,353529,1424,1326,399766,1395949742,8639936081,624,0,0,0,201771890,13,15767,14,3,15657,3,15657,0,31314,19008,432,16 1,67,1,97957,0,386,386,386,386.0,60,1,1,381,0,0,0,0,0,0,138414192,0,5343,154,221,9724,210,9240,0,18964,44,1,0 -1,56,431,89600,0,616500,1899,589,1430.4,378074,1330,1515,612098,16461920726,8639936081,4132,0,0,0,216740322,23,15805,10,3,15657,3,15657,0,31314,18964,431,16 +1,56,431,89600,0,616500,1899,589,1430.4,378287,1521,1515,612098,16461920726,8639936081,4132,0,0,0,216740322,23,15805,10,3,15657,3,15657,0,31314,18964,431,16 1,70,1,71716,0,384,384,384,384.0,54,1,1,379,0,0,0,0,0,0,138414192,0,5343,170,223,9812,210,9240,0,19052,44,1,0 -1,52,431,51060,0,759623,2321,918,1762.5,394996,1460,2027,754015,26337468742,8639936081,7772,0,0,0,250648581,87,16157,170,3,15657,3,15657,0,31314,18964,431,19 +1,52,431,51060,0,759623,2321,918,1762.5,395214,1706,2027,754015,26337468742,8639936081,7772,0,0,0,250648581,87,16157,170,3,15657,3,15657,0,31314,18964,431,19 1,73,1,46297,0,136,136,136,136.0,57,1,1,131,0,0,0,0,0,0,138414192,0,5344,0,214,9416,219,9636,0,19052,44,1,0 1,76,1,23048,0,340,340,340,340.0,36,1,1,334,0,0,0,0,0,0,138414192,0,5343,223,215,9460,217,9548,0,19008,44,1,0 1,31,1,6956,0,6738,6738,6738,6738.0,5104,128,688,6035,349526,86400,53,0,0,0,155563380,1,10759,0,0,0,0,0,0,0,7239,1800,0 @@ -19,20 +19,20 @@ appIndex,stageId,numTasks,Duration,diskBytesSpilled_sum,duration_sum,duration_ma 1,34,1,6907,0,6729,6729,6729,6729.0,157,136,681,6035,12261,1350,53,0,0,0,155199546,1,9839,0,0,0,0,0,0,0,699,165,0 1,38,1,6842,0,6743,6743,6743,6743.0,187,256,688,6035,349526,86400,53,0,0,0,155563380,1,9927,0,0,0,0,0,0,0,7239,1800,0 1,0,1,5904,0,5699,5699,5699,5699.0,422,948,1114,4382,0,0,37,0,0,0,0,8,2794,0,0,0,0,0,0,0,0,0,0 -1,13,200,5697,0,87661,966,349,438.3,9821,427,951,84265,0,0,144,0,0,0,0,9,6258,0,0,0,0,0,0,0,0,0,0 -1,23,200,5476,0,84240,490,355,421.2,5290,200,214,82784,0,0,136,0,0,0,0,0,6214,0,0,0,0,0,0,0,0,0,0 -1,21,200,5265,0,80904,485,353,404.5,6004,203,220,79384,0,0,136,0,0,0,0,1,6302,0,0,0,0,0,0,0,0,0,0 -1,27,200,4719,0,70760,442,309,353.8,4042,200,209,69494,0,0,152,0,0,0,0,10,5788,0,0,0,0,0,0,0,0,0,0 +1,13,200,5697,0,87661,966,349,438.3,9924,528,951,84265,0,0,144,0,0,0,0,9,6258,0,0,0,0,0,0,0,0,0,0 +1,23,200,5476,0,84240,490,355,421.2,5394,292,214,82784,0,0,136,0,0,0,0,0,6214,0,0,0,0,0,0,0,0,0,0 +1,21,200,5265,0,80904,485,353,404.5,6100,304,220,79384,0,0,136,0,0,0,0,1,6302,0,0,0,0,0,0,0,0,0,0 +1,27,200,4719,0,70760,442,309,353.8,4145,287,209,69494,0,0,152,0,0,0,0,10,5788,0,0,0,0,0,0,0,0,0,0 1,3,1,4701,0,4693,4693,4693,4693.0,280,701,804,3796,0,0,26,0,0,0,0,7,2834,0,0,0,0,0,0,0,0,0,0 -1,25,200,4599,0,70379,569,314,351.9,4106,200,216,69040,0,0,168,0,0,0,0,14,5708,0,0,0,0,0,0,0,0,0,0 -1,29,200,4552,0,69682,423,310,348.4,3730,200,218,68521,0,0,168,0,0,0,0,9,5748,0,0,0,0,0,0,0,0,0,0 +1,25,200,4599,0,70379,569,314,351.9,4200,294,216,69040,0,0,168,0,0,0,0,14,5708,0,0,0,0,0,0,0,0,0,0 +1,29,200,4552,0,69682,423,310,348.4,3830,272,218,68521,0,0,168,0,0,0,0,9,5748,0,0,0,0,0,0,0,0,0,0 1,35,1,4525,0,4332,4332,4332,4332.0,3359,95,401,3907,30328,7200,39,0,0,0,155245068,1,10552,0,0,0,0,0,0,0,7719,1920,0 1,36,1,4509,0,4334,4334,4334,4334.0,260,130,404,3907,349526,86400,39,0,0,0,155563380,1,9851,0,0,0,0,0,0,0,7239,1800,0 1,39,1,4474,0,4322,4322,4322,4322.0,112,124,392,3907,349526,86400,39,0,0,0,155563380,1,9926,0,0,0,0,0,0,0,7239,1800,0 1,40,1,4469,0,4327,4327,4327,4327.0,98,147,394,3907,349526,86400,39,0,0,0,155563380,1,9895,0,0,0,0,0,0,0,7239,1800,0 1,37,1,4464,0,4334,4334,4334,4334.0,136,144,405,3907,349526,86400,39,0,0,0,155563380,1,9851,0,0,0,0,0,0,0,7239,1800,0 1,107,1,1052,0,1022,1022,1022,1022.0,758,77,95,901,0,0,0,0,0,0,134218344,6,10091,5,218,9592,220,9680,0,19272,0,0,0 -1,19,200,794,0,11895,145,38,59.5,943,209,252,10017,0,0,56,0,0,0,0,22,2739,0,0,0,0,0,0,0,0,0,0 +1,19,200,794,0,11895,145,38,59.5,1050,321,252,10017,0,0,56,0,0,0,0,22,2739,0,0,0,0,0,0,0,0,0,0 1,26,1,315,0,312,312,312,312.0,2,1,1,306,0,0,0,0,0,0,0,0,3777,0,0,0,0,0,0,0,0,0,0 1,2,1,276,0,267,267,267,267.0,6,4,4,124,0,0,0,0,0,0,0,1,3342,0,0,0,0,0,0,0,0,0,0 1,11,1,260,0,254,254,254,254.0,5,3,3,241,0,0,0,0,0,0,0,0,2913,0,0,0,0,0,0,0,0,0,0 diff --git a/core/src/test/resources/ProfilingExpectations/rapids_duration_and_cpu_expectation.csv b/core/src/test/resources/ProfilingExpectations/rapids_duration_and_cpu_expectation.csv index ebde48473..2aad941a0 100644 --- a/core/src/test/resources/ProfilingExpectations/rapids_duration_and_cpu_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/rapids_duration_and_cpu_expectation.csv @@ -1,9 +1,9 @@ appIndex,App ID,rootSqlId,sqlID,SQL Duration,Contains Dataset or RDD Op,App Duration,Potential Problems,Executor CPU Time Percent -1,"local-1626104300434","",0,1260,false,131104,"NESTED COMPLEX TYPE",92.65 -1,"local-1626104300434","",1,259,false,131104,"NESTED COMPLEX TYPE",76.79 -1,"local-1626104300434","",2,130,false,131104,"NESTED COMPLEX TYPE",90.48 -1,"local-1626104300434","",3,76,false,131104,"NESTED COMPLEX TYPE",97.56 -1,"local-1626104300434","",4,65,false,131104,"NESTED COMPLEX TYPE",100.0 -1,"local-1626104300434","",5,479,false,131104,"NESTED COMPLEX TYPE",87.32 +1,"local-1626104300434","",0,1260,false,131104,"NESTED COMPLEX TYPE",93.01 +1,"local-1626104300434","",1,259,false,131104,"NESTED COMPLEX TYPE",77.38 +1,"local-1626104300434","",2,130,false,131104,"NESTED COMPLEX TYPE",92.06 +1,"local-1626104300434","",3,76,false,131104,"NESTED COMPLEX TYPE",100.0 +1,"local-1626104300434","",4,65,false,131104,"NESTED COMPLEX TYPE",103.45 +1,"local-1626104300434","",5,479,false,131104,"NESTED COMPLEX TYPE",87.8 1,"local-1626104300434","",6,95,false,131104,"",96.3 1,"local-1626104300434","",7,65,false,131104,"",95.24 diff --git a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobmetricsagg2_expectation.csv b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobmetricsagg2_expectation.csv index 3d1553c6d..655179961 100644 --- a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobmetricsagg2_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobmetricsagg2_expectation.csv @@ -1,2 +1,2 @@ appIndex,jobId,numTasks,Duration,diskBytesSpilled_sum,duration_sum,duration_max,duration_min,duration_avg,executorCPUTime_sum,executorDeserializeCPUTime_sum,executorDeserializeTime_sum,executorRunTime_sum,input_bytesRead_sum,input_recordsRead_sum,jvmGCTime_sum,memoryBytesSpilled_sum,output_bytesWritten_sum,output_recordsWritten_sum,peakExecutionMemory_max,resultSerializationTime_sum,resultSize_max,sr_fetchWaitTime_sum,sr_localBlocksFetched_sum,sr_localBytesRead_sum,sr_remoteBlocksFetched_sum,sr_remoteBytesRead_sum,sr_remoteBytesReadToDisk_sum,sr_totalBytesRead_sum,sw_bytesWritten_sum,sw_recordsWritten_sum,sw_writeTime_sum -1,0,213,2515,0,25761,1624,9,120.9,7045,3021,11178,13522,0,0,424,0,0,0,0,10,8075,0,2600,80279920,0,0,0,80279920,80279920,2600,901 +1,0,213,2515,0,25761,1624,9,120.9,7151,3134,11178,13522,0,0,424,0,0,0,0,10,8075,0,2600,80279920,0,0,0,80279920,80279920,2600,901 diff --git a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobmetricsagg_expectation.csv b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobmetricsagg_expectation.csv index 02d56dcab..b4cd7175c 100644 --- a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobmetricsagg_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobmetricsagg_expectation.csv @@ -1,2 +1,2 @@ appIndex,jobId,numTasks,Duration,diskBytesSpilled_sum,duration_sum,duration_max,duration_min,duration_avg,executorCPUTime_sum,executorDeserializeCPUTime_sum,executorDeserializeTime_sum,executorRunTime_sum,input_bytesRead_sum,input_recordsRead_sum,jvmGCTime_sum,memoryBytesSpilled_sum,output_bytesWritten_sum,output_recordsWritten_sum,peakExecutionMemory_max,resultSerializationTime_sum,resultSize_max,sr_fetchWaitTime_sum,sr_localBlocksFetched_sum,sr_localBytesRead_sum,sr_remoteBlocksFetched_sum,sr_remoteBytesRead_sum,sr_remoteBytesReadToDisk_sum,sr_totalBytesRead_sum,sw_bytesWritten_sum,sw_recordsWritten_sum,sw_writeTime_sum -1,0,213,2569,0,26735,1598,10,125.5,6500,3433,12095,13414,0,0,336,0,0,0,0,8,8075,0,2600,80279908,0,0,0,80279908,80279908,2600,1001 +1,0,213,2569,0,26735,1598,10,125.5,6608,3531,12095,13414,0,0,336,0,0,0,0,8,8075,0,2600,80279908,0,0,0,80279908,80279908,2600,1001 diff --git a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobmetricsaggmulti_expectation.csv b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobmetricsaggmulti_expectation.csv index 4135857b1..d9202e3ad 100644 --- a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobmetricsaggmulti_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_jobmetricsaggmulti_expectation.csv @@ -1,3 +1,3 @@ appIndex,jobId,numTasks,Duration,diskBytesSpilled_sum,duration_sum,duration_max,duration_min,duration_avg,executorCPUTime_sum,executorDeserializeCPUTime_sum,executorDeserializeTime_sum,executorRunTime_sum,input_bytesRead_sum,input_recordsRead_sum,jvmGCTime_sum,memoryBytesSpilled_sum,output_bytesWritten_sum,output_recordsWritten_sum,peakExecutionMemory_max,resultSerializationTime_sum,resultSize_max,sr_fetchWaitTime_sum,sr_localBlocksFetched_sum,sr_localBytesRead_sum,sr_remoteBlocksFetched_sum,sr_remoteBytesRead_sum,sr_remoteBytesReadToDisk_sum,sr_totalBytesRead_sum,sw_bytesWritten_sum,sw_recordsWritten_sum,sw_writeTime_sum -1,0,213,2569,0,26735,1598,10,125.5,6500,3433,12095,13414,0,0,336,0,0,0,0,8,8075,0,2600,80279908,0,0,0,80279908,80279908,2600,1001 -2,0,213,2515,0,25761,1624,9,120.9,7045,3021,11178,13522,0,0,424,0,0,0,0,10,8075,0,2600,80279920,0,0,0,80279920,80279920,2600,901 +1,0,213,2569,0,26735,1598,10,125.5,6608,3531,12095,13414,0,0,336,0,0,0,0,8,8075,0,2600,80279908,0,0,0,80279908,80279908,2600,1001 +2,0,213,2515,0,25761,1624,9,120.9,7151,3134,11178,13522,0,0,424,0,0,0,0,10,8075,0,2600,80279920,0,0,0,80279920,80279920,2600,901 diff --git a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg2_expectation.csv b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg2_expectation.csv index e6cbb84e7..b4fa1dff0 100644 --- a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg2_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg2_expectation.csv @@ -1,2 +1,2 @@ appIndex,appID,sqlID,description,numTasks,Duration,executorCPUTime,executorRunTime,executorCPURatio,diskBytesSpilled_sum,duration_sum,duration_max,duration_min,duration_avg,executorCPUTime_sum,executorDeserializeCPUTime_sum,executorDeserializeTime_sum,executorRunTime_sum,input_bytesRead_sum,input_recordsRead_sum,jvmGCTime_sum,memoryBytesSpilled_sum,output_bytesWritten_sum,output_recordsWritten_sum,peakExecutionMemory_max,resultSerializationTime_sum,resultSize_max,sr_fetchWaitTime_sum,sr_localBlocksFetched_sum,sr_localBytesRead_sum,sr_remoteBlocksFetched_sum,sr_remoteBytesRead_sum,sr_remoteBytesReadToDisk_sum,sr_totalBytesRead_sum,sw_bytesWritten_sum,sw_recordsWritten_sum,sw_writeTime_sum -1,"local-1622821994212",0,"count at :28",213,3041,7045,13522,52.1,0,25761,1624,9,120.9,7045,3021,11178,13522,0,0,424,0,0,0,0,10,8075,0,2600,80279920,0,0,0,80279920,80279920,2600,901 +1,"local-1622821994212",0,"count at :28",213,3041,7151,13522,52.88,0,25761,1624,9,120.9,7151,3134,11178,13522,0,0,424,0,0,0,0,10,8075,0,2600,80279920,0,0,0,80279920,80279920,2600,901 diff --git a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg_expectation.csv b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg_expectation.csv index 4e777dad3..1cf88cdb6 100644 --- a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsagg_expectation.csv @@ -1,2 +1,2 @@ appIndex,appID,sqlID,description,numTasks,Duration,executorCPUTime,executorRunTime,executorCPURatio,diskBytesSpilled_sum,duration_sum,duration_max,duration_min,duration_avg,executorCPUTime_sum,executorDeserializeCPUTime_sum,executorDeserializeTime_sum,executorRunTime_sum,input_bytesRead_sum,input_recordsRead_sum,jvmGCTime_sum,memoryBytesSpilled_sum,output_bytesWritten_sum,output_recordsWritten_sum,peakExecutionMemory_max,resultSerializationTime_sum,resultSize_max,sr_fetchWaitTime_sum,sr_localBlocksFetched_sum,sr_localBytesRead_sum,sr_remoteBlocksFetched_sum,sr_remoteBytesRead_sum,sr_remoteBytesReadToDisk_sum,sr_totalBytesRead_sum,sw_bytesWritten_sum,sw_recordsWritten_sum,sw_writeTime_sum -1,"local-1622814619968",0,"count at :28",213,3087,6500,13414,48.46,0,26735,1598,10,125.5,6500,3433,12095,13414,0,0,336,0,0,0,0,8,8075,0,2600,80279908,0,0,0,80279908,80279908,2600,1001 +1,"local-1622814619968",0,"count at :28",213,3087,6608,13414,49.26,0,26735,1598,10,125.5,6608,3531,12095,13414,0,0,336,0,0,0,0,8,8075,0,2600,80279908,0,0,0,80279908,80279908,2600,1001 diff --git a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsaggmulti_expectation.csv b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsaggmulti_expectation.csv index 29a7e7316..fab4b6a71 100644 --- a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsaggmulti_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetricsaggmulti_expectation.csv @@ -1,3 +1,3 @@ appIndex,appID,sqlID,description,numTasks,Duration,executorCPUTime,executorRunTime,executorCPURatio,diskBytesSpilled_sum,duration_sum,duration_max,duration_min,duration_avg,executorCPUTime_sum,executorDeserializeCPUTime_sum,executorDeserializeTime_sum,executorRunTime_sum,input_bytesRead_sum,input_recordsRead_sum,jvmGCTime_sum,memoryBytesSpilled_sum,output_bytesWritten_sum,output_recordsWritten_sum,peakExecutionMemory_max,resultSerializationTime_sum,resultSize_max,sr_fetchWaitTime_sum,sr_localBlocksFetched_sum,sr_localBytesRead_sum,sr_remoteBlocksFetched_sum,sr_remoteBytesRead_sum,sr_remoteBytesReadToDisk_sum,sr_totalBytesRead_sum,sw_bytesWritten_sum,sw_recordsWritten_sum,sw_writeTime_sum -1,"local-1622814619968",0,"count at :28",213,3087,6500,13414,48.46,0,26735,1598,10,125.5,6500,3433,12095,13414,0,0,336,0,0,0,0,8,8075,0,2600,80279908,0,0,0,80279908,80279908,2600,1001 -2,"local-1622821994212",0,"count at :28",213,3041,7045,13522,52.1,0,25761,1624,9,120.9,7045,3021,11178,13522,0,0,424,0,0,0,0,10,8075,0,2600,80279920,0,0,0,80279920,80279920,2600,901 +1,"local-1622814619968",0,"count at :28",213,3087,6608,13414,49.26,0,26735,1598,10,125.5,6608,3531,12095,13414,0,0,336,0,0,0,0,8,8075,0,2600,80279908,0,0,0,80279908,80279908,2600,1001 +2,"local-1622821994212",0,"count at :28",213,3041,7151,13522,52.88,0,25761,1624,9,120.9,7151,3134,11178,13522,0,0,424,0,0,0,0,10,8075,0,2600,80279920,0,0,0,80279920,80279920,2600,901 diff --git a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_stagemetricsagg2_expectation.csv b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_stagemetricsagg2_expectation.csv index bfccc7960..4aab14955 100644 --- a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_stagemetricsagg2_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_stagemetricsagg2_expectation.csv @@ -1,5 +1,5 @@ appIndex,stageId,numTasks,Duration,diskBytesSpilled_sum,duration_sum,duration_max,duration_min,duration_avg,executorCPUTime_sum,executorDeserializeCPUTime_sum,executorDeserializeTime_sum,executorRunTime_sum,input_bytesRead_sum,input_recordsRead_sum,jvmGCTime_sum,memoryBytesSpilled_sum,output_bytesWritten_sum,output_recordsWritten_sum,peakExecutionMemory_max,resultSerializationTime_sum,resultSize_max,sr_fetchWaitTime_sum,sr_localBlocksFetched_sum,sr_localBytesRead_sum,sr_remoteBlocksFetched_sum,sr_remoteBytesRead_sum,sr_remoteBytesReadToDisk_sum,sr_totalBytesRead_sum,sw_bytesWritten_sum,sw_recordsWritten_sum,sw_writeTime_sum -1,0,6,1761,0,9455,1624,1540,1575.8,2914,1283,5056,4248,0,0,228,0,0,0,0,3,2951,0,0,0,0,0,0,0,40132263,1200,376 -1,1,6,1666,0,9274,1621,1528,1545.7,2568,1004,5016,4099,0,0,196,0,0,0,0,4,2951,0,0,0,0,0,0,0,40132257,1200,475 -1,2,200,592,0,6937,221,9,34.7,1518,696,1065,5125,0,0,0,0,0,0,0,3,7402,0,2400,80264520,0,0,0,80264520,15400,200,50 +1,0,6,1761,0,9455,1624,1540,1575.8,2917,1287,5056,4248,0,0,228,0,0,0,0,3,2951,0,0,0,0,0,0,0,40132263,1200,376 +1,1,6,1666,0,9274,1621,1528,1545.7,2570,1007,5016,4099,0,0,196,0,0,0,0,4,2951,0,0,0,0,0,0,0,40132257,1200,475 +1,2,200,592,0,6937,221,9,34.7,1619,802,1065,5125,0,0,0,0,0,0,0,3,7402,0,2400,80264520,0,0,0,80264520,15400,200,50 1,3,1,101,0,95,95,95,95.0,45,38,41,50,0,0,0,0,0,0,0,0,8075,0,200,15400,0,0,0,15400,0,0,0 diff --git a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_stagemetricsagg_expectation.csv b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_stagemetricsagg_expectation.csv index 997cc22af..b99aebc5a 100644 --- a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_stagemetricsagg_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_stagemetricsagg_expectation.csv @@ -1,5 +1,5 @@ appIndex,stageId,numTasks,Duration,diskBytesSpilled_sum,duration_sum,duration_max,duration_min,duration_avg,executorCPUTime_sum,executorDeserializeCPUTime_sum,executorDeserializeTime_sum,executorRunTime_sum,input_bytesRead_sum,input_recordsRead_sum,jvmGCTime_sum,memoryBytesSpilled_sum,output_bytesWritten_sum,output_recordsWritten_sum,peakExecutionMemory_max,resultSerializationTime_sum,resultSize_max,sr_fetchWaitTime_sum,sr_localBlocksFetched_sum,sr_localBytesRead_sum,sr_remoteBlocksFetched_sum,sr_remoteBytesRead_sum,sr_remoteBytesReadToDisk_sum,sr_totalBytesRead_sum,sw_bytesWritten_sum,sw_recordsWritten_sum,sw_writeTime_sum -1,0,6,1743,0,9518,1598,1580,1586.3,2509,1391,5309,4043,0,0,168,0,0,0,0,3,2951,0,0,0,0,0,0,0,40132250,1200,400 -1,1,6,1631,0,9434,1582,1568,1572.3,2405,1065,5273,3998,0,0,168,0,0,0,0,5,2951,0,0,0,0,0,0,0,40132258,1200,508 -1,2,200,688,0,7705,237,10,38.5,1556,940,1474,5337,0,0,0,0,0,0,0,0,7359,0,2400,80264508,0,0,0,80264508,15400,200,93 +1,0,6,1743,0,9518,1598,1580,1586.3,2512,1393,5309,4043,0,0,168,0,0,0,0,3,2951,0,0,0,0,0,0,0,40132250,1200,400 +1,1,6,1631,0,9434,1582,1568,1572.3,2406,1067,5273,3998,0,0,168,0,0,0,0,5,2951,0,0,0,0,0,0,0,40132258,1200,508 +1,2,200,688,0,7705,237,10,38.5,1660,1034,1474,5337,0,0,0,0,0,0,0,0,7359,0,2400,80264508,0,0,0,80264508,15400,200,93 1,3,1,83,0,78,78,78,78.0,30,37,39,36,0,0,0,0,0,0,0,0,8075,0,200,15400,0,0,0,15400,0,0,0 diff --git a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_stagemetricsaggmulti_expectation.csv b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_stagemetricsaggmulti_expectation.csv index b8624e1d7..4704cafa9 100644 --- a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_stagemetricsaggmulti_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_stagemetricsaggmulti_expectation.csv @@ -1,9 +1,9 @@ appIndex,stageId,numTasks,Duration,diskBytesSpilled_sum,duration_sum,duration_max,duration_min,duration_avg,executorCPUTime_sum,executorDeserializeCPUTime_sum,executorDeserializeTime_sum,executorRunTime_sum,input_bytesRead_sum,input_recordsRead_sum,jvmGCTime_sum,memoryBytesSpilled_sum,output_bytesWritten_sum,output_recordsWritten_sum,peakExecutionMemory_max,resultSerializationTime_sum,resultSize_max,sr_fetchWaitTime_sum,sr_localBlocksFetched_sum,sr_localBytesRead_sum,sr_remoteBlocksFetched_sum,sr_remoteBytesRead_sum,sr_remoteBytesReadToDisk_sum,sr_totalBytesRead_sum,sw_bytesWritten_sum,sw_recordsWritten_sum,sw_writeTime_sum -1,0,6,1743,0,9518,1598,1580,1586.3,2509,1391,5309,4043,0,0,168,0,0,0,0,3,2951,0,0,0,0,0,0,0,40132250,1200,400 -1,1,6,1631,0,9434,1582,1568,1572.3,2405,1065,5273,3998,0,0,168,0,0,0,0,5,2951,0,0,0,0,0,0,0,40132258,1200,508 -1,2,200,688,0,7705,237,10,38.5,1556,940,1474,5337,0,0,0,0,0,0,0,0,7359,0,2400,80264508,0,0,0,80264508,15400,200,93 +1,0,6,1743,0,9518,1598,1580,1586.3,2512,1393,5309,4043,0,0,168,0,0,0,0,3,2951,0,0,0,0,0,0,0,40132250,1200,400 +1,1,6,1631,0,9434,1582,1568,1572.3,2406,1067,5273,3998,0,0,168,0,0,0,0,5,2951,0,0,0,0,0,0,0,40132258,1200,508 +1,2,200,688,0,7705,237,10,38.5,1660,1034,1474,5337,0,0,0,0,0,0,0,0,7359,0,2400,80264508,0,0,0,80264508,15400,200,93 1,3,1,83,0,78,78,78,78.0,30,37,39,36,0,0,0,0,0,0,0,0,8075,0,200,15400,0,0,0,15400,0,0,0 -2,0,6,1761,0,9455,1624,1540,1575.8,2914,1283,5056,4248,0,0,228,0,0,0,0,3,2951,0,0,0,0,0,0,0,40132263,1200,376 -2,1,6,1666,0,9274,1621,1528,1545.7,2568,1004,5016,4099,0,0,196,0,0,0,0,4,2951,0,0,0,0,0,0,0,40132257,1200,475 -2,2,200,592,0,6937,221,9,34.7,1518,696,1065,5125,0,0,0,0,0,0,0,3,7402,0,2400,80264520,0,0,0,80264520,15400,200,50 +2,0,6,1761,0,9455,1624,1540,1575.8,2917,1287,5056,4248,0,0,228,0,0,0,0,3,2951,0,0,0,0,0,0,0,40132263,1200,376 +2,1,6,1666,0,9274,1621,1528,1545.7,2570,1007,5016,4099,0,0,196,0,0,0,0,4,2951,0,0,0,0,0,0,0,40132257,1200,475 +2,2,200,592,0,6937,221,9,34.7,1619,802,1065,5125,0,0,0,0,0,0,0,3,7402,0,2400,80264520,0,0,0,80264520,15400,200,50 2,3,1,101,0,95,95,95,95.0,45,38,41,50,0,0,0,0,0,0,0,0,8075,0,200,15400,0,0,0,15400,0,0,0