Skip to content

Commit

Permalink
addressed review comments
Browse files Browse the repository at this point in the history
  • Loading branch information
nartal1 committed Aug 23, 2023
1 parent a52803a commit 7b6803f
Show file tree
Hide file tree
Showing 5 changed files with 22 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ class Qualification(outputDir: String, numRows: Int, hadoopConf: Configuration,
timeout: Option[Long], nThreads: Int, order: String,
pluginTypeChecker: PluginTypeChecker, reportReadSchema: Boolean,
printStdout: Boolean, uiEnabled: Boolean, enablePB: Boolean,
reportSqlLevel: Boolean, maxSQLDescLength: Int, mlOpsEnabled:Boolean) extends Logging {
reportSqlLevel: Boolean, maxSQLDescLength: Int, mlOpsEnabled:Boolean,
ignoreTransitions: Boolean) extends Logging {

private val allApps = new ConcurrentLinkedQueue[QualificationSummaryInfo]()

Expand Down Expand Up @@ -166,7 +167,7 @@ class Qualification(outputDir: String, numRows: Int, hadoopConf: Configuration,
try {
val startTime = System.currentTimeMillis()
val appResult = QualificationAppInfo.createApp(path, hadoopConf, pluginTypeChecker,
reportSqlLevel, mlOpsEnabled)
reportSqlLevel, mlOpsEnabled, ignoreTransitions)
val qualAppResult = appResult match {
case Left(errorMessage: String) =>
// Case when an error occurred during QualificationAppInfo creation
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,11 @@ Usage: java -cp rapids-4-spark-tools_2.12-<version>.jar:$SPARK_HOME/jars/*
opt[Boolean](required = false,
descr = "Whether to parse ML functions in the eventlogs. Default is false.",
default = Some(false))
val ignoreTransitions: ScallopOption[Boolean] =
opt[Boolean](required = false,
descr = "Whether to ignore durations for ColumnarToRow and RowToColumnar transitions " +
"in the eventlogs while calculating the speedup. Default is false.",
default = Some(false))
val sparkProperty: ScallopOption[List[String]] =
opt[List[String]](required = false,
descr = "Filter applications based on certain Spark properties that were set during " +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ object QualificationMain extends Logging {
val reportSqlLevel = appArgs.perSql.getOrElse(false)
val platform = appArgs.platform.getOrElse("onprem")
val mlOpsEnabled = appArgs.mlFunctions.getOrElse(false)
val ignoreTransitions = appArgs.ignoreTransitions.getOrElse(false)

val hadoopConf = RapidsToolsConfUtil.newHadoopConf

Expand Down Expand Up @@ -93,7 +94,7 @@ object QualificationMain extends Logging {

val qual = new Qualification(outputDirectory, numOutputRows, hadoopConf, timeout,
nThreads, order, pluginTypeChecker, reportReadSchema, printStdout, uiEnabled,
enablePB, reportSqlLevel, maxSQLDescLength, mlOpsEnabled)
enablePB, reportSqlLevel, maxSQLDescLength, mlOpsEnabled, ignoreTransitions)
val res = qual.qualifyApps(filteredLogs)
(0, res)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ class QualificationAppInfo(
pluginTypeChecker: PluginTypeChecker,
reportSqlLevel: Boolean,
perSqlOnly: Boolean = false,
mlOpsEnabled: Boolean = false)
mlOpsEnabled: Boolean = false,
ignoreTransitions: Boolean = false)
extends AppBase(eventLogInfo, hadoopConf) with Logging {

var appId: String = ""
Expand Down Expand Up @@ -262,8 +263,13 @@ class QualificationAppInfo(
stages.map { stageId =>
val stageTaskTime = stageIdToTaskEndSum.get(stageId)
.map(_.totalTaskDuration).getOrElse(0L)
val numTransitions = stageIdToGpuCpuTransitions.getOrElse(stageId, 0)
val numTransitions = ignoreTransitions match {
case false => stageIdToGpuCpuTransitions.getOrElse(stageId, 0)
case true => 0
}
// val numTransitions = stageIdToGpuCpuTransitions.getOrElse(stageId, 0)
val transitionsTime = numTransitions match {
case 0 => 0L // no transitions
case gpuCpuTransitions if gpuCpuTransitions > 0 =>
// Duration to transfer data from GPU to CPU and vice versa.
// Assuming it's a PCI-E Gen3, but also assuming that some of the result could be
Expand Down Expand Up @@ -929,10 +935,11 @@ object QualificationAppInfo extends Logging {
hadoopConf: Configuration,
pluginTypeChecker: PluginTypeChecker,
reportSqlLevel: Boolean,
mlOpsEnabled: Boolean): Either[String, QualificationAppInfo] = {
mlOpsEnabled: Boolean,
ignoreTransitions: Boolean): Either[String, QualificationAppInfo] = {
try {
val app = new QualificationAppInfo(Some(path), Some(hadoopConf), pluginTypeChecker,
reportSqlLevel, false, mlOpsEnabled)
reportSqlLevel, false, mlOpsEnabled, ignoreTransitions)
logInfo(s"${path.eventLog.toString} has App: ${app.appId}")
Right(app)
} catch {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ class SQLPlanParserSuite extends BaseTestSuite {
val pluginTypeChecker = new PluginTypeChecker()
assert(allEventLogs.size == 1)
val appResult = QualificationAppInfo.createApp(allEventLogs.head, hadoopConf,
pluginTypeChecker, reportSqlLevel = false, mlOpsEnabled = false)
pluginTypeChecker, reportSqlLevel = false, mlOpsEnabled = false, ignoreTransitions = false)
appResult match {
case Right(app) => app
case Left(_) => throw new AssertionError("Cannot create application")
Expand Down

0 comments on commit 7b6803f

Please sign in to comment.