diff --git a/tools/databricks/[RAPIDS Accelerator for Apache Spark] Qualification User Tool Notebook Template.ipynb b/tools/databricks/[RAPIDS Accelerator for Apache Spark] Qualification User Tool Notebook Template.ipynb new file mode 100644 index 000000000..32b3e39f6 --- /dev/null +++ b/tools/databricks/[RAPIDS Accelerator for Apache Spark] Qualification User Tool Notebook Template.ipynb @@ -0,0 +1,3868 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "df33c614-2ecc-47a0-8600-bc891681997f", + "showTitle": false, + "title": "" + } + }, + "source": [ + "# Welcome to the Qualification User Tool for the RAPIDS Accelerator for Apache Spark\n", + "To run the user tool, you need to enter a log path that represents the DBFS location for your Spark CPU event logs. Then you can select \"Run all\" to execute the notebook. After the notebook completes, you will see various output tables show up below. More options for running the qualification user tool can be found here: https://docs.nvidia.com/spark-rapids/user-guide/latest/spark-qualification-tool.html#running-the-qualification-tool-standalone-for-csp-environments-on-spark-event-logs.\n", + "\n", + "## Summary Output\n", + "The report represents the entire app execution, including unsupported operators and non-SQL operations. By default, the applications and queries are sorted in descending order by the following fields:\n", + "- Recommendation;\n", + "- Estimated GPU Speed-up;\n", + "- Estimated GPU Time Saved; and\n", + "- End Time.\n", + "\n", + "## Stages Output\n", + "For each stage used in SQL operations, the Qualification tool generates the following information:\n", + "1. App ID\n", + "1. Stage ID\n", + "1. Average Speedup Factor: the average estimated speed-up of all the operators in the given stage.\n", + "1. Stage Task Duration: amount of time spent in tasks of SQL Dataframe operations for the given stage.\n", + "1. Unsupported Task Duration: sum of task durations for the unsupported operators. For more details, see Supported Operators.\n", + "1. Stage Estimated: True or False indicates if we had to estimate the stage duration.\n", + "\n", + "## Execs Output\n", + "The Qualification tool generates a report of the “Exec” in the “SparkPlan” or “Executor Nodes” along with the estimated acceleration on the GPU. Please refer to the Supported Operators guide for more details on limitations on UDFs and unsupported operators.\n", + "1. App ID\n", + "1. SQL ID\n", + "1. Exec Name: example Filter, HashAggregate\n", + "1. Expression Name\n", + "1. Task Speedup Factor: it is simply the average acceleration of the operators based on the original CPU duration of the operator divided by the GPU duration. The tool uses historical queries and benchmarks to estimate a speed-up at an individual operator level to calculate how much a specific operator would accelerate on GPU.\n", + "1. Exec Duration: wall-Clock time measured since the operator starts till it is completed.\n", + "1. SQL Node Id\n", + "1. Exec Is Supported: whether the Exec is supported by RAPIDS or not. Please refer to the Supported Operators section.\n", + "1. Exec Stages: an array of stage IDs\n", + "1. Exec Children\n", + "1. Exec Children Node Ids\n", + "1. Exec Should Remove: whether the Op is removed from the migrated plan." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "5ed51af2-b7ae-46b1-ba9c-ebbcc23e24f2", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[43mNote: you may need to restart the kernel using dbutils.library.restartPython() to use updated packages.\u001B[0m\n\u001B[43mNote: you may need to restart the kernel using dbutils.library.restartPython() to use updated packages.\u001B[0m\n" + ] + } + ], + "source": [ + "pip install spark-rapids-user-tools > /dev/null" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "53b4d770-9db6-4bd7-9b93-d036d375eac5", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "dbutils.library.restartPython() " + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "acf401a3-12d3-4236-a6c5-8fe8990b153a", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "dbutils.widgets.text(\"log_path\", \"\")\n", + "eventlog_string=dbutils.widgets.get(\"log_path\")\n", + "\n", + "dbutils.widgets.text(\"output_path\", \"\")\n", + "outputpath_string=dbutils.widgets.get(\"output_path\")\n", + "\n", + "dbutils.widgets.dropdown(\"csp\", \"aws\", [\"aws\", \"azure\"])\n", + "csp_string=dbutils.widgets.get(\"csp\")\n", + "platform_string=f\"databricks-{csp_string}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "693b5ee0-7500-43f3-b3e2-717fd5468aa8", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "output_type": "stream", + "text": [ + "\r\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽\rProcessing...⣾\rProcessing...⣷\rProcessing...⣯\rProcessing...⣟\rProcessing...⡿\rProcessing...⢿\rProcessing...⣻\rProcessing...⣽" + ] + } + ], + "source": [ + "!spark_rapids_user_tools $platform_string qualification --eventlogs $eventlog_string --local_folder $outputpath_string > /tmp/qual_debug.log" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "bbe50fde-0bd6-4281-95fd-6a1ec6f17ab2", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## Summary Output" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "fb8edb26-e173-47ff-92a1-463baec7c06b", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": [ + "
App NameApp IDRecommendationEstimated GPU SpeedupEstimated GPU DurationEstimated GPU Time SavedSQL DF DurationSQL Dataframe Task DurationApp DurationGPU OpportunityExecutor CPU Time PercentSQL Ids with FailuresUnsupported Read File Formats and TypesUnsupported Write Data FormatComplex TypesNested Complex TypesPotential ProblemsLongest SQL DurationNONSQL Task Duration Plus OverheadUnsupported Task DurationSupported SQL DF Task DurationTask Speedup FactorApp Duration EstimatedUnsupported ExecsUnsupported ExpressionsEstimated Job Frequency (monthly)
TPC-DS Like Bench q4app-20220209224316-0007Strongly Recommended2.7840078.1171498.8890167853247001115779016765.09nullnullnullnullnullnull9016330865160853247004.83falseExecute CreateViewCommandnull30
" + ] + }, + "metadata": { + "application/vnd.databricks.v1+output": { + "addedWidgets": {}, + "aggData": [], + "aggError": "", + "aggOverflow": false, + "aggSchema": [], + "aggSeriesLimitReached": false, + "aggType": "", + "arguments": {}, + "columnCustomDisplayInfos": {}, + "data": [ + [ + "TPC-DS Like Bench q4", + "app-20220209224316-0007", + "Strongly Recommended", + 2.78, + 40078.11, + 71498.88, + 90167, + 85324700, + 111577, + 90167, + 65.09, + null, + null, + null, + null, + null, + null, + 90163, + 3086516, + 0, + 85324700, + 4.83, + false, + "Execute CreateViewCommand", + null, + 30 + ] + ], + "datasetInfos": [], + "dbfsResultPath": null, + "isJsonSchema": true, + "metadata": {}, + "overflow": false, + "plotOptions": { + "customPlotOptions": {}, + "displayType": "table", + "pivotAggregation": null, + "pivotColumns": null, + "xColumns": null, + "yColumns": null + }, + "removedWidgets": [], + "schema": [ + { + "metadata": "{}", + "name": "App Name", + "type": "\"string\"" + }, + { + "metadata": "{}", + "name": "App ID", + "type": "\"string\"" + }, + { + "metadata": "{}", + "name": "Recommendation", + "type": "\"string\"" + }, + { + "metadata": "{}", + "name": "Estimated GPU Speedup", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "Estimated GPU Duration", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "Estimated GPU Time Saved", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "SQL DF Duration", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "SQL Dataframe Task Duration", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "App Duration", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "GPU Opportunity", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "Executor CPU Time Percent", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "SQL Ids with Failures", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "Unsupported Read File Formats and Types", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "Unsupported Write Data Format", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "Complex Types", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "Nested Complex Types", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "Potential Problems", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "Longest SQL Duration", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "NONSQL Task Duration Plus Overhead", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "Unsupported Task Duration", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "Supported SQL DF Task Duration", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "Task Speedup Factor", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "App Duration Estimated", + "type": "\"boolean\"" + }, + { + "metadata": "{}", + "name": "Unsupported Execs", + "type": "\"string\"" + }, + { + "metadata": "{}", + "name": "Unsupported Expressions", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "Estimated Job Frequency (monthly)", + "type": "\"long\"" + } + ], + "type": "table" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "summary_output=pd.read_csv(outputpath_string + \"/rapids_4_spark_qualification_output/rapids_4_spark_qualification_output.csv\")\n", + "display(summary_output)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "6756159b-30ca-407a-ab6b-9c29ced01ea6", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## Stages Output" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "cdde6177-db5f-434a-995b-776678a64a3a", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": [ + "
App IDStage IDAverage Speedup FactorStage Task DurationUnsupported Task DurationStage EstimatedNumber of transitions from or to GPU
app-20220209224316-0007455.91133492860false0
app-20220209224316-0007364.138056800false0
app-20220209224316-0007424.133934560false0
app-20220209224316-0007484.744877560false0
app-20220209224316-0007415.9198503010false0
app-20220209224316-0007313.7611860false0
app-20220209224316-0007494.755670230false0
app-20220209224316-0007404.137332100false0
app-20220209224316-0007323.767070false0
app-20220209224316-0007528.0727475850false0
app-20220209224316-0007344.132793850false0
app-20220209224316-0007334.1318303060false0
app-20220209224316-0007384.1311277860false0
app-20220209224316-0007444.1324787970false0
app-20220209224316-0007474.594310550false0
app-20220209224316-0007435.9150181140false0
app-20220209224316-0007375.9154483780false0
app-20220209224316-0007533.02410false0
app-20220209224316-0007355.91195730460false0
app-20220209224316-0007514.745153690false0
app-20220209224316-0007395.91184024460false0
app-20220209224316-0007504.745126400false0
app-20220209224316-0007464.597709470false0
" + ] + }, + "metadata": { + "application/vnd.databricks.v1+output": { + "addedWidgets": {}, + "aggData": [], + "aggError": "", + "aggOverflow": false, + "aggSchema": [], + "aggSeriesLimitReached": false, + "aggType": "", + "arguments": {}, + "columnCustomDisplayInfos": {}, + "data": [ + [ + "app-20220209224316-0007", + 45, + 5.91, + 13349286, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 36, + 4.13, + 805680, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 42, + 4.13, + 393456, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 48, + 4.74, + 487756, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 41, + 5.91, + 9850301, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 31, + 3.76, + 1186, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 49, + 4.75, + 567023, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 40, + 4.13, + 733210, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 32, + 3.76, + 707, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 52, + 8.07, + 2747585, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 34, + 4.13, + 279385, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 33, + 4.13, + 1830306, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 38, + 4.13, + 1127786, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 44, + 4.13, + 2478797, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 47, + 4.59, + 431055, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 43, + 5.91, + 5018114, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 37, + 5.91, + 5448378, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 53, + 3.0, + 241, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 35, + 5.91, + 19573046, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 51, + 4.74, + 515369, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 39, + 5.91, + 18402446, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 50, + 4.74, + 512640, + 0, + false, + 0 + ], + [ + "app-20220209224316-0007", + 46, + 4.59, + 770947, + 0, + false, + 0 + ] + ], + "datasetInfos": [], + "dbfsResultPath": null, + "isJsonSchema": true, + "metadata": {}, + "overflow": false, + "plotOptions": { + "customPlotOptions": {}, + "displayType": "table", + "pivotAggregation": null, + "pivotColumns": null, + "xColumns": null, + "yColumns": null + }, + "removedWidgets": [], + "schema": [ + { + "metadata": "{}", + "name": "App ID", + "type": "\"string\"" + }, + { + "metadata": "{}", + "name": "Stage ID", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "Average Speedup Factor", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "Stage Task Duration", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "Unsupported Task Duration", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "Stage Estimated", + "type": "\"boolean\"" + }, + { + "metadata": "{}", + "name": "Number of transitions from or to GPU", + "type": "\"long\"" + } + ], + "type": "table" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "stages_output=pd.read_csv(outputpath_string + \"/rapids_4_spark_qualification_output/rapids_4_spark_qualification_output_stages.csv\")\n", + "display(stages_output)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "4d7ce219-ae75-4a0c-a78c-4e7f25b8cd6f", + "showTitle": false, + "title": "" + } + }, + "source": [ + "## Execs Output" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "998b0c51-0cb6-408e-a01a-d1f5b1a61e1f", + "showTitle": false, + "title": "" + } + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": [ + "
App IDSQL IDExec NameExpression NameTask Speedup FactorExec DurationSQL Node IdExec Is SupportedExec StagesExec ChildrenExec Children Node IdsExec Should Remove
app-20220209224316-000724SortMergeJoinnull22.330101true45nullnullfalse
app-20220209224316-000724WholeStageCodegen (14)WholeStageCodegen (14)7.491879093761true35HashAggregate:Project:BroadcastHashJoin:Project:SortMergeJoin62:63:64:65:66false
app-20220209224316-000724SubqueryBroadcastnull3.097643truenullnullnullfalse
app-20220209224316-000724Exchangenull4.912696160true50:52nullnullfalse
app-20220209224316-000724WholeStageCodegen (2)WholeStageCodegen (2)6.641750827629true39Sort30false
app-20220209224316-000724HashAggregatenull4.41024true39nullnullfalse
app-20220209224316-000724Sortnull6.640113true52nullnullfalse
app-20220209224316-000724WholeStageCodegen (10)WholeStageCodegen (10)6.641877215667true35Sort68false
app-20220209224316-000724ReusedExchangenull1.0077truenullnullnulltrue
app-20220209224316-000724HashAggregatenull4.410165true37nullnullfalse
app-20220209224316-000724WholeStageCodegen (12)WholeStageCodegen (12)6.641931342669true35Sort70false
app-20220209224316-000724SortMergeJoinnull22.330169true37nullnullfalse
app-20220209224316-000724WholeStageCodegen (15)WholeStageCodegen (15)4.4136359658true51HashAggregate59false
app-20220209224316-000724Sortnull6.640149true43nullnullfalse
app-20220209224316-000724Scan parquetnull3.030154true32nullnullfalse
app-20220209224316-00074Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724WholeStageCodegen (26)WholeStageCodegen (26)12.67156932111true52Project:SortMergeJoin12:13false
app-20220209224316-000724Exchangenull4.931434106true44:45nullnullfalse
app-20220209224316-000724WholeStageCodegen (21)WholeStageCodegen (21)6.6413059736104true45Sort105false
app-20220209224316-000724Projectnull3.006truenullnullnullfalse
app-20220209224316-000724BroadcastExchangenull3.0138578truenullnullnulltrue
app-20220209224316-000724SortMergeJoinnull22.330123true41nullnullfalse
app-20220209224316-000724ColumnarToRownull1.00109false44nullnulltrue
app-20220209224316-000724Scan parquetnull3.054582true31nullnulltrue
app-20220209224316-000724Exchangenull4.932062174true36:37nullnullfalse
app-20220209224316-000724WholeStageCodegen (42)WholeStageCodegen (42)3.97322554137true47Project:Filter:HashAggregate138:139:140false
app-20220209224316-000724WholeStageCodegen (23)WholeStageCodegen (23)7.491213414996true45HashAggregate:Project:BroadcastHashJoin:Project:SortMergeJoin97:98:99:100:101false
app-20220209224316-000720Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724WholeStageCodegen (24)WholeStageCodegen (24)3.9749069091true46Project:Filter:HashAggregate92:93:94false
app-20220209224316-000724WholeStageCodegen (35)WholeStageCodegen (35)12.6716378378true52Project:SortMergeJoin9:10false
app-20220209224316-000717Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724BroadcastExchangenull3.094750truenullnullnullfalse
app-20220209224316-000724Projectnull3.00144truenullnullnullfalse
app-20220209224316-000724SubqueryBroadcastnull3.0976111truenullnullnullfalse
app-20220209224316-000724Sortnull6.640127true41nullnullfalse
app-20220209224316-000724WholeStageCodegen (52)WholeStageCodegen (52)6.64836330158true52Sort159false
app-20220209224316-000724BroadcastHashJoinnull4.720121true41nullnullfalse
app-20220209224316-000724Projectnull3.00146truenullnullnullfalse
app-20220209224316-000724TakeOrderedAndProjectnull3.001true52:53nullnullfalse
app-20220209224316-000724Sortnull6.640125true41nullnullfalse
app-20220209224316-000724SubqueryBroadcastnull3.0976157truenullnullnullfalse
app-20220209224316-000724WholeStageCodegen (7)WholeStageCodegen (7)4.4635244819true49Filter:HashAggregate20:21false
app-20220209224316-000724Scan parquetnull3.0193671178true36nullnullfalse
app-20220209224316-000724ColumnarToRownull1.0086false31nullnulltrue
app-20220209224316-000721Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-00071Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724Filternull4.51052true32nullnullfalse
app-20220209224316-000724WholeStageCodegen (5)WholeStageCodegen (5)4.5132051true32Filter:ColumnarToRow52:53false
app-20220209224316-000724WholeStageCodegen (53)WholeStageCodegen (53)12.6720916342true52Project:SortMergeJoin3:4false
app-20220209224316-000724Sortnull6.64089true52nullnullfalse
app-20220209224316-000724Scan parquetnull3.038922275true33nullnullfalse
app-20220209224316-000724BroadcastHashJoinnull4.720167true37nullnullfalse
app-20220209224316-000724BroadcastExchangenull3.0138583truenullnullnullfalse
app-20220209224316-000724BroadcastHashJoinnull4.72099true45nullnullfalse
app-20220209224316-000724WholeStageCodegen (19)WholeStageCodegen (19)6.6411861853102true45Sort103false
app-20220209224316-000724Sortnull6.640105true45nullnullfalse
app-20220209224316-000724Exchangenull4.9156049117true41:48nullnullfalse
app-20220209224316-000724Projectnull3.00168truenullnullnullfalse
app-20220209224316-000724Exchangenull4.920924060true35:51nullnullfalse
app-20220209224316-000724SubqueryBroadcastnull3.01388179truenullnullnullfalse
app-20220209224316-000724Projectnull3.00138truenullnullnullfalse
app-20220209224316-000724Scan parquetnull3.054587true31nullnullfalse
app-20220209224316-000724Sortnull6.64068true35nullnullfalse
app-20220209224316-000724HashAggregatenull4.41021true49nullnullfalse
app-20220209224316-000724Exchangenull4.98460136true47:52nullnullfalse
app-20220209224316-000724WholeStageCodegen (51)WholeStageCodegen (51)4.41314233161true50HashAggregate162false
app-20220209224316-000724Projectnull3.00166truenullnullnullfalse
app-20220209224316-000724ColumnarToRownull1.00131false40nullnulltrue
app-20220209224316-00073Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724ColumnarToRownull1.0034false34nullnulltrue
app-20220209224316-000724Filternull4.51020true49nullnullfalse
app-20220209224316-00078Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724WholeStageCodegen (11)WholeStageCodegen (11)4.51135307772true33Filter:ColumnarToRow73:74false
app-20220209224316-000724ReusedExchangenull1.0044truenullnullnulltrue
app-20220209224316-000724Execute InsertIntoHadoopFsRelationCommand parquetnull3.000truenullnullnullfalse
app-20220209224316-000724ColumnarToRownull1.0041false38nullnulltrue
app-20220209224316-000724HashAggregatenull4.410143true43nullnullfalse
app-20220209224316-000724Exchangenull4.96123557true51:52nullnullfalse
app-20220209224316-000724SortMergeJoinnull22.33013true52nullnullfalse
app-20220209224316-000724Filternull4.510108true44nullnullfalse
app-20220209224316-000712Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724WholeStageCodegen (33)WholeStageCodegen (33)4.41311678115true48HashAggregate116false
app-20220209224316-000724ColumnarToRownull1.0081false31nullnulltrue
app-20220209224316-000724WholeStageCodegen (32)WholeStageCodegen (32)7.499334072118true41HashAggregate:Project:BroadcastHashJoin:Project:SortMergeJoin119:120:121:122:123false
app-20220209224316-000724WholeStageCodegen (20)WholeStageCodegen (20)4.511602599107true44Filter:ColumnarToRow108:109false
app-20220209224316-000724Scan parquetnull3.027149342true38nullnullfalse
app-20220209224316-000724Sortnull6.64070true35nullnullfalse
app-20220209224316-000724HashAggregatenull4.410119true41nullnullfalse
app-20220209224316-000724WholeStageCodegen (44)WholeStageCodegen (44)12.6719304875true52Project:SortMergeJoin6:7false
app-20220209224316-000724Exchangenull4.915288114true48:52nullnullfalse
app-20220209224316-000724WholeStageCodegen (47)WholeStageCodegen (47)4.51531329175true36Filter:ColumnarToRow176:177false
app-20220209224316-000724WholeStageCodegen (8)WholeStageCodegen (8)6.64149443116true52Sort17false
app-20220209224316-000724HashAggregatenull4.41059true51nullnullfalse
app-20220209224316-000724Projectnull3.0098truenullnullnullfalse
app-20220209224316-000722Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724Exchangenull4.93517690true46:52nullnullfalse
app-20220209224316-000724WholeStageCodegen (43)WholeStageCodegen (43)6.641250915134true52Sort135false
app-20220209224316-000724Filternull4.510176true36nullnullfalse
app-20220209224316-000724HashAggregatenull4.410140true47nullnullfalse
app-20220209224316-000724Exchangenull4.921251671true33:35nullnullfalse
app-20220209224316-000724Exchangenull4.920686128true40:41nullnullfalse
app-20220209224316-000724Scan parquetnull3.0832627110true44nullnullfalse
app-20220209224316-000724Sortnull6.64030true39nullnullfalse
app-20220209224316-000724WholeStageCodegen (1)WholeStageCodegen (1)4.5115564432true34Filter:ColumnarToRow33:34false
app-20220209224316-000724BroadcastHashJoinnull4.72026true39nullnullfalse
app-20220209224316-000724Scan parquetnull3.010265035true34nullnullfalse
app-20220209224316-000724Exchangenull4.9162543163true37:50nullnullfalse
app-20220209224316-000724WholeStageCodegen (29)WholeStageCodegen (29)4.51530409129true40Filter:ColumnarToRow130:131false
app-20220209224316-000724Sortnull6.640159true52nullnullfalse
app-20220209224316-00076Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724Filternull4.51093true46nullnullfalse
app-20220209224316-000724SortMergeJoinnull22.33015true52nullnullfalse
app-20220209224316-000724Projectnull3.0063truenullnullnullfalse
app-20220209224316-000724WholeStageCodegen (16)WholeStageCodegen (16)6.64219726855true52Sort56false
app-20220209224316-000724Scan parquetnull3.030149true32nullnulltrue
app-20220209224316-000724WholeStageCodegen (50)WholeStageCodegen (50)7.494913238164true37HashAggregate:Project:BroadcastHashJoin:Project:SortMergeJoin165:166:167:168:169false
app-20220209224316-000724WholeStageCodegen (13)WholeStageCodegen (13)4.5156584true31Filter:ColumnarToRow85:86false
app-20220209224316-000724Projectnull3.0012truenullnullnullfalse
app-20220209224316-000724WholeStageCodegen (28)WholeStageCodegen (28)6.648723134124true41Sort125false
app-20220209224316-00070Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724HashAggregatenull4.41062true35nullnullfalse
app-20220209224316-000724Exchangenull4.9200753141true43:47nullnullfalse
app-20220209224316-000724Filternull4.51040true38nullnullfalse
app-20220209224316-000724HashAggregatenull4.410116true48nullnullfalse
app-20220209224316-000724BroadcastHashJoinnull4.72064true35nullnullfalse
app-20220209224316-000724SortMergeJoinnull22.3304true52nullnullfalse
app-20220209224316-000724Exchangenull4.920935395true45:46nullnullfalse
app-20220209224316-000724SubqueryBroadcastnull3.0138876truenullnullnullfalse
app-20220209224316-000724WholeStageCodegen (17)WholeStageCodegen (17)22.33144680014true52SortMergeJoin15false
app-20220209224316-000724Exchangenull4.914608522true39:49nullnullfalse
app-20220209224316-000724HashAggregatenull4.41094true46nullnullfalse
app-20220209224316-000724Sortnull6.640173true37nullnullfalse
app-20220209224316-000724Projectnull3.0025truenullnullnullfalse
app-20220209224316-000724Projectnull3.003truenullnullnullfalse
app-20220209224316-000724HashAggregatenull4.410162true50nullnullfalse
app-20220209224316-000724Projectnull3.00122truenullnullnullfalse
app-20220209224316-000724ColumnarToRownull1.00155false42nullnulltrue
app-20220209224316-000724Scan parquetnull3.0216336132true40nullnullfalse
app-20220209224316-000724Projectnull3.0065truenullnullnullfalse
app-20220209224316-000724WholeStageCodegen (46)WholeStageCodegen (46)6.643372368170true37Sort171false
app-20220209224316-000724Projectnull3.00100truenullnullnullfalse
app-20220209224316-000724SubqueryBroadcastnull3.01388133truenullnullnullfalse
app-20220209224316-000715Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724WholeStageCodegen (6)WholeStageCodegen (6)7.491777808923true39HashAggregate:Project:BroadcastHashJoin:Project:SortMergeJoin24:25:26:27:28false
app-20220209224316-000724Filternull4.51033true34nullnullfalse
app-20220209224316-000714Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724Sortnull6.64037true39nullnullfalse
app-20220209224316-000716Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724Filternull4.51047true32nullnulltrue
app-20220209224316-000724BroadcastExchangenull3.094745truenullnullnulltrue
app-20220209224316-000724ColumnarToRownull1.0074false33nullnulltrue
app-20220209224316-000724HashAggregatenull4.41097true45nullnullfalse
app-20220209224316-000724Exchangenull4.923981152true42:43nullnullfalse
app-20220209224316-000724Filternull4.510139true47nullnullfalse
app-20220209224316-00075Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724Sortnull6.64017true52nullnullfalse
app-20220209224316-000724WholeStageCodegen (41)WholeStageCodegen (41)7.494639507142true43HashAggregate:Project:BroadcastHashJoin:Project:SortMergeJoin143:144:145:146:147false
app-20220209224316-000724WholeStageCodegen (48)WholeStageCodegen (48)6.646648471172true37Sort173false
app-20220209224316-000724WholeStageCodegen (37)WholeStageCodegen (37)6.642909792148true43Sort149false
app-20220209224316-000724Sortnull6.640103true45nullnullfalse
app-20220209224316-000724BroadcastHashJoinnull4.720145true43nullnullfalse
app-20220209224316-000724Projectnull3.0027truenullnullnullfalse
app-20220209224316-000724SortMergeJoinnull22.33010true52nullnullfalse
app-20220209224316-000724WholeStageCodegen (34)WholeStageCodegen (34)6.641990423112true52Sort113false
app-20220209224316-00072Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-00077Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724WholeStageCodegen (5)WholeStageCodegen (5)1.032046true32Filter:ColumnarToRow47:48true
app-20220209224316-000724WholeStageCodegen (13)WholeStageCodegen (13)1.056579true31Filter:ColumnarToRow80:81true
app-20220209224316-000724WholeStageCodegen (25)WholeStageCodegen (25)6.64226215288true52Sort89false
app-20220209224316-000724ColumnarToRownull1.00177false36nullnulltrue
app-20220209224316-000724WholeStageCodegen (30)WholeStageCodegen (30)6.6410069460126true41Sort127false
app-20220209224316-000724ColumnarToRownull1.0048false32nullnulltrue
app-20220209224316-000724SortMergeJoinnull22.33028true39nullnullfalse
app-20220209224316-000724Exchangenull4.96886418true49:52nullnullfalse
app-20220209224316-00079Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724Sortnull6.640135true52nullnullfalse
app-20220209224316-000724ColumnarToRownull1.0053false32nullnulltrue
app-20220209224316-000724Exchangenull4.92987838true38:39nullnullfalse
app-20220209224316-000710Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724Filternull4.510130true40nullnullfalse
app-20220209224316-000724SortMergeJoinnull22.33066true35nullnullfalse
app-20220209224316-000724Projectnull3.00120truenullnullnullfalse
app-20220209224316-000724WholeStageCodegen (4)WholeStageCodegen (4)6.641825100136true39Sort37false
app-20220209224316-000724Exchangenull4.93848931true37:41:34:45:39:35:43nullnullfalse
app-20220209224316-000724Scan parquetnull3.0118333156true42nullnullfalse
app-20220209224316-000718Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724Sortnull6.640171true37nullnullfalse
app-20220209224316-000724Filternull4.510154true42nullnullfalse
app-20220209224316-000723Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724Filternull4.51080true31nullnulltrue
app-20220209224316-000724WholeStageCodegen (39)WholeStageCodegen (39)6.646477320150true43Sort151false
app-20220209224316-000724Sortnull6.64056true52nullnullfalse
app-20220209224316-000719Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724Filternull4.51073true33nullnullfalse
app-20220209224316-000724Projectnull3.009truenullnullnullfalse
app-20220209224316-000724WholeStageCodegen (3)WholeStageCodegen (3)4.5183329639true38Filter:ColumnarToRow40:41false
app-20220209224316-000724Filternull4.51085true31nullnullfalse
app-20220209224316-000724Projectnull3.0092truenullnullnullfalse
app-20220209224316-000724Sortnull6.640151true43nullnullfalse
app-20220209224316-000724WholeStageCodegen (38)WholeStageCodegen (38)4.51289863153true42Filter:ColumnarToRow154:155false
app-20220209224316-000711Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724SortMergeJoinnull22.330147true43nullnullfalse
app-20220209224316-000713Execute CreateViewCommandnull1.000falsenullnullnullfalse
app-20220209224316-000724SortMergeJoinnull22.3307true52nullnullfalse
" + ] + }, + "metadata": { + "application/vnd.databricks.v1+output": { + "addedWidgets": {}, + "aggData": [], + "aggError": "", + "aggOverflow": false, + "aggSchema": [], + "aggSeriesLimitReached": false, + "aggType": "", + "arguments": {}, + "columnCustomDisplayInfos": {}, + "data": [ + [ + "app-20220209224316-0007", + 24, + "SortMergeJoin", + null, + 22.33, + 0, + 101, + true, + "45", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (14)", + "WholeStageCodegen (14)", + 7.49, + 18790937, + 61, + true, + "35", + "HashAggregate:Project:BroadcastHashJoin:Project:SortMergeJoin", + "62:63:64:65:66", + false + ], + [ + "app-20220209224316-0007", + 24, + "SubqueryBroadcast", + null, + 3.0, + 976, + 43, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 12696, + 160, + true, + "50:52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (2)", + "WholeStageCodegen (2)", + 6.64, + 17508276, + 29, + true, + "39", + "Sort", + "30", + false + ], + [ + "app-20220209224316-0007", + 24, + "HashAggregate", + null, + 4.41, + 0, + 24, + true, + "39", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 113, + true, + "52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (10)", + "WholeStageCodegen (10)", + 6.64, + 18772156, + 67, + true, + "35", + "Sort", + "68", + false + ], + [ + "app-20220209224316-0007", + 24, + "ReusedExchange", + null, + 1.0, + 0, + 77, + true, + null, + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "HashAggregate", + null, + 4.41, + 0, + 165, + true, + "37", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (12)", + "WholeStageCodegen (12)", + 6.64, + 19313426, + 69, + true, + "35", + "Sort", + "70", + false + ], + [ + "app-20220209224316-0007", + 24, + "SortMergeJoin", + null, + 22.33, + 0, + 169, + true, + "37", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (15)", + "WholeStageCodegen (15)", + 4.41, + 363596, + 58, + true, + "51", + "HashAggregate", + "59", + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 149, + true, + "43", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Scan parquet", + null, + 3.0, + 301, + 54, + true, + "32", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 4, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (26)", + "WholeStageCodegen (26)", + 12.67, + 1569321, + 11, + true, + "52", + "Project:SortMergeJoin", + "12:13", + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 31434, + 106, + true, + "44:45", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (21)", + "WholeStageCodegen (21)", + 6.64, + 13059736, + 104, + true, + "45", + "Sort", + "105", + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 6, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "BroadcastExchange", + null, + 3.0, + 1385, + 78, + true, + null, + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "SortMergeJoin", + null, + 22.33, + 0, + 123, + true, + "41", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "ColumnarToRow", + null, + 1.0, + 0, + 109, + false, + "44", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "Scan parquet", + null, + 3.0, + 545, + 82, + true, + "31", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 32062, + 174, + true, + "36:37", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (42)", + "WholeStageCodegen (42)", + 3.97, + 322554, + 137, + true, + "47", + "Project:Filter:HashAggregate", + "138:139:140", + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (23)", + "WholeStageCodegen (23)", + 7.49, + 12134149, + 96, + true, + "45", + "HashAggregate:Project:BroadcastHashJoin:Project:SortMergeJoin", + "97:98:99:100:101", + false + ], + [ + "app-20220209224316-0007", + 20, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (24)", + "WholeStageCodegen (24)", + 3.97, + 490690, + 91, + true, + "46", + "Project:Filter:HashAggregate", + "92:93:94", + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (35)", + "WholeStageCodegen (35)", + 12.67, + 1637837, + 8, + true, + "52", + "Project:SortMergeJoin", + "9:10", + false + ], + [ + "app-20220209224316-0007", + 17, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "BroadcastExchange", + null, + 3.0, + 947, + 50, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 144, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "SubqueryBroadcast", + null, + 3.0, + 976, + 111, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 127, + true, + "41", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (52)", + "WholeStageCodegen (52)", + 6.64, + 836330, + 158, + true, + "52", + "Sort", + "159", + false + ], + [ + "app-20220209224316-0007", + 24, + "BroadcastHashJoin", + null, + 4.72, + 0, + 121, + true, + "41", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 146, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "TakeOrderedAndProject", + null, + 3.0, + 0, + 1, + true, + "52:53", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 125, + true, + "41", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "SubqueryBroadcast", + null, + 3.0, + 976, + 157, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (7)", + "WholeStageCodegen (7)", + 4.46, + 352448, + 19, + true, + "49", + "Filter:HashAggregate", + "20:21", + false + ], + [ + "app-20220209224316-0007", + 24, + "Scan parquet", + null, + 3.0, + 193671, + 178, + true, + "36", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "ColumnarToRow", + null, + 1.0, + 0, + 86, + false, + "31", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 21, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 1, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 52, + true, + "32", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (5)", + "WholeStageCodegen (5)", + 4.51, + 320, + 51, + true, + "32", + "Filter:ColumnarToRow", + "52:53", + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (53)", + "WholeStageCodegen (53)", + 12.67, + 2091634, + 2, + true, + "52", + "Project:SortMergeJoin", + "3:4", + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 89, + true, + "52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Scan parquet", + null, + 3.0, + 389222, + 75, + true, + "33", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "BroadcastHashJoin", + null, + 4.72, + 0, + 167, + true, + "37", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "BroadcastExchange", + null, + 3.0, + 1385, + 83, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "BroadcastHashJoin", + null, + 4.72, + 0, + 99, + true, + "45", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (19)", + "WholeStageCodegen (19)", + 6.64, + 11861853, + 102, + true, + "45", + "Sort", + "103", + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 105, + true, + "45", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 156049, + 117, + true, + "41:48", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 168, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 209240, + 60, + true, + "35:51", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "SubqueryBroadcast", + null, + 3.0, + 1388, + 179, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 138, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Scan parquet", + null, + 3.0, + 545, + 87, + true, + "31", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 68, + true, + "35", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "HashAggregate", + null, + 4.41, + 0, + 21, + true, + "49", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 8460, + 136, + true, + "47:52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (51)", + "WholeStageCodegen (51)", + 4.41, + 314233, + 161, + true, + "50", + "HashAggregate", + "162", + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 166, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "ColumnarToRow", + null, + 1.0, + 0, + 131, + false, + "40", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 3, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "ColumnarToRow", + null, + 1.0, + 0, + 34, + false, + "34", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 20, + true, + "49", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 8, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (11)", + "WholeStageCodegen (11)", + 4.51, + 1353077, + 72, + true, + "33", + "Filter:ColumnarToRow", + "73:74", + false + ], + [ + "app-20220209224316-0007", + 24, + "ReusedExchange", + null, + 1.0, + 0, + 44, + true, + null, + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "Execute InsertIntoHadoopFsRelationCommand parquet", + null, + 3.0, + 0, + 0, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "ColumnarToRow", + null, + 1.0, + 0, + 41, + false, + "38", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "HashAggregate", + null, + 4.41, + 0, + 143, + true, + "43", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 61235, + 57, + true, + "51:52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "SortMergeJoin", + null, + 22.33, + 0, + 13, + true, + "52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 108, + true, + "44", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 12, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (33)", + "WholeStageCodegen (33)", + 4.41, + 311678, + 115, + true, + "48", + "HashAggregate", + "116", + false + ], + [ + "app-20220209224316-0007", + 24, + "ColumnarToRow", + null, + 1.0, + 0, + 81, + false, + "31", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (32)", + "WholeStageCodegen (32)", + 7.49, + 9334072, + 118, + true, + "41", + "HashAggregate:Project:BroadcastHashJoin:Project:SortMergeJoin", + "119:120:121:122:123", + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (20)", + "WholeStageCodegen (20)", + 4.51, + 1602599, + 107, + true, + "44", + "Filter:ColumnarToRow", + "108:109", + false + ], + [ + "app-20220209224316-0007", + 24, + "Scan parquet", + null, + 3.0, + 271493, + 42, + true, + "38", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 70, + true, + "35", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "HashAggregate", + null, + 4.41, + 0, + 119, + true, + "41", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (44)", + "WholeStageCodegen (44)", + 12.67, + 1930487, + 5, + true, + "52", + "Project:SortMergeJoin", + "6:7", + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 15288, + 114, + true, + "48:52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (47)", + "WholeStageCodegen (47)", + 4.51, + 531329, + 175, + true, + "36", + "Filter:ColumnarToRow", + "176:177", + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (8)", + "WholeStageCodegen (8)", + 6.64, + 1494431, + 16, + true, + "52", + "Sort", + "17", + false + ], + [ + "app-20220209224316-0007", + 24, + "HashAggregate", + null, + 4.41, + 0, + 59, + true, + "51", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 98, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 22, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 35176, + 90, + true, + "46:52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (43)", + "WholeStageCodegen (43)", + 6.64, + 1250915, + 134, + true, + "52", + "Sort", + "135", + false + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 176, + true, + "36", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "HashAggregate", + null, + 4.41, + 0, + 140, + true, + "47", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 212516, + 71, + true, + "33:35", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 20686, + 128, + true, + "40:41", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Scan parquet", + null, + 3.0, + 832627, + 110, + true, + "44", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 30, + true, + "39", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (1)", + "WholeStageCodegen (1)", + 4.51, + 155644, + 32, + true, + "34", + "Filter:ColumnarToRow", + "33:34", + false + ], + [ + "app-20220209224316-0007", + 24, + "BroadcastHashJoin", + null, + 4.72, + 0, + 26, + true, + "39", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Scan parquet", + null, + 3.0, + 102650, + 35, + true, + "34", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 162543, + 163, + true, + "37:50", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (29)", + "WholeStageCodegen (29)", + 4.51, + 530409, + 129, + true, + "40", + "Filter:ColumnarToRow", + "130:131", + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 159, + true, + "52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 6, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 93, + true, + "46", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "SortMergeJoin", + null, + 22.33, + 0, + 15, + true, + "52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 63, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (16)", + "WholeStageCodegen (16)", + 6.64, + 2197268, + 55, + true, + "52", + "Sort", + "56", + false + ], + [ + "app-20220209224316-0007", + 24, + "Scan parquet", + null, + 3.0, + 301, + 49, + true, + "32", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (50)", + "WholeStageCodegen (50)", + 7.49, + 4913238, + 164, + true, + "37", + "HashAggregate:Project:BroadcastHashJoin:Project:SortMergeJoin", + "165:166:167:168:169", + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (13)", + "WholeStageCodegen (13)", + 4.51, + 565, + 84, + true, + "31", + "Filter:ColumnarToRow", + "85:86", + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 12, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (28)", + "WholeStageCodegen (28)", + 6.64, + 8723134, + 124, + true, + "41", + "Sort", + "125", + false + ], + [ + "app-20220209224316-0007", + 0, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "HashAggregate", + null, + 4.41, + 0, + 62, + true, + "35", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 200753, + 141, + true, + "43:47", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 40, + true, + "38", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "HashAggregate", + null, + 4.41, + 0, + 116, + true, + "48", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "BroadcastHashJoin", + null, + 4.72, + 0, + 64, + true, + "35", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "SortMergeJoin", + null, + 22.33, + 0, + 4, + true, + "52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 209353, + 95, + true, + "45:46", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "SubqueryBroadcast", + null, + 3.0, + 1388, + 76, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (17)", + "WholeStageCodegen (17)", + 22.33, + 1446800, + 14, + true, + "52", + "SortMergeJoin", + "15", + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 146085, + 22, + true, + "39:49", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "HashAggregate", + null, + 4.41, + 0, + 94, + true, + "46", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 173, + true, + "37", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 25, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 3, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "HashAggregate", + null, + 4.41, + 0, + 162, + true, + "50", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 122, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "ColumnarToRow", + null, + 1.0, + 0, + 155, + false, + "42", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "Scan parquet", + null, + 3.0, + 216336, + 132, + true, + "40", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 65, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (46)", + "WholeStageCodegen (46)", + 6.64, + 3372368, + 170, + true, + "37", + "Sort", + "171", + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 100, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "SubqueryBroadcast", + null, + 3.0, + 1388, + 133, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 15, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (6)", + "WholeStageCodegen (6)", + 7.49, + 17778089, + 23, + true, + "39", + "HashAggregate:Project:BroadcastHashJoin:Project:SortMergeJoin", + "24:25:26:27:28", + false + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 33, + true, + "34", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 14, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 37, + true, + "39", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 16, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 47, + true, + "32", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "BroadcastExchange", + null, + 3.0, + 947, + 45, + true, + null, + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "ColumnarToRow", + null, + 1.0, + 0, + 74, + false, + "33", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "HashAggregate", + null, + 4.41, + 0, + 97, + true, + "45", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 23981, + 152, + true, + "42:43", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 139, + true, + "47", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 5, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 17, + true, + "52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (41)", + "WholeStageCodegen (41)", + 7.49, + 4639507, + 142, + true, + "43", + "HashAggregate:Project:BroadcastHashJoin:Project:SortMergeJoin", + "143:144:145:146:147", + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (48)", + "WholeStageCodegen (48)", + 6.64, + 6648471, + 172, + true, + "37", + "Sort", + "173", + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (37)", + "WholeStageCodegen (37)", + 6.64, + 2909792, + 148, + true, + "43", + "Sort", + "149", + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 103, + true, + "45", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "BroadcastHashJoin", + null, + 4.72, + 0, + 145, + true, + "43", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 27, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "SortMergeJoin", + null, + 22.33, + 0, + 10, + true, + "52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (34)", + "WholeStageCodegen (34)", + 6.64, + 1990423, + 112, + true, + "52", + "Sort", + "113", + false + ], + [ + "app-20220209224316-0007", + 2, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 7, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (5)", + "WholeStageCodegen (5)", + 1.0, + 320, + 46, + true, + "32", + "Filter:ColumnarToRow", + "47:48", + true + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (13)", + "WholeStageCodegen (13)", + 1.0, + 565, + 79, + true, + "31", + "Filter:ColumnarToRow", + "80:81", + true + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (25)", + "WholeStageCodegen (25)", + 6.64, + 2262152, + 88, + true, + "52", + "Sort", + "89", + false + ], + [ + "app-20220209224316-0007", + 24, + "ColumnarToRow", + null, + 1.0, + 0, + 177, + false, + "36", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (30)", + "WholeStageCodegen (30)", + 6.64, + 10069460, + 126, + true, + "41", + "Sort", + "127", + false + ], + [ + "app-20220209224316-0007", + 24, + "ColumnarToRow", + null, + 1.0, + 0, + 48, + false, + "32", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "SortMergeJoin", + null, + 22.33, + 0, + 28, + true, + "39", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 68864, + 18, + true, + "49:52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 9, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 135, + true, + "52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "ColumnarToRow", + null, + 1.0, + 0, + 53, + false, + "32", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 29878, + 38, + true, + "38:39", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 10, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 130, + true, + "40", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "SortMergeJoin", + null, + 22.33, + 0, + 66, + true, + "35", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 120, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (4)", + "WholeStageCodegen (4)", + 6.64, + 18251001, + 36, + true, + "39", + "Sort", + "37", + false + ], + [ + "app-20220209224316-0007", + 24, + "Exchange", + null, + 4.9, + 38489, + 31, + true, + "37:41:34:45:39:35:43", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Scan parquet", + null, + 3.0, + 118333, + 156, + true, + "42", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 18, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 171, + true, + "37", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 154, + true, + "42", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 23, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 80, + true, + "31", + null, + null, + true + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (39)", + "WholeStageCodegen (39)", + 6.64, + 6477320, + 150, + true, + "43", + "Sort", + "151", + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 56, + true, + "52", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 19, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 73, + true, + "33", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 9, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (3)", + "WholeStageCodegen (3)", + 4.51, + 833296, + 39, + true, + "38", + "Filter:ColumnarToRow", + "40:41", + false + ], + [ + "app-20220209224316-0007", + 24, + "Filter", + null, + 4.51, + 0, + 85, + true, + "31", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Project", + null, + 3.0, + 0, + 92, + true, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "Sort", + null, + 6.64, + 0, + 151, + true, + "43", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "WholeStageCodegen (38)", + "WholeStageCodegen (38)", + 4.51, + 289863, + 153, + true, + "42", + "Filter:ColumnarToRow", + "154:155", + false + ], + [ + "app-20220209224316-0007", + 11, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "SortMergeJoin", + null, + 22.33, + 0, + 147, + true, + "43", + null, + null, + false + ], + [ + "app-20220209224316-0007", + 13, + "Execute CreateViewCommand", + null, + 1.0, + 0, + 0, + false, + null, + null, + null, + false + ], + [ + "app-20220209224316-0007", + 24, + "SortMergeJoin", + null, + 22.33, + 0, + 7, + true, + "52", + null, + null, + false + ] + ], + "datasetInfos": [], + "dbfsResultPath": null, + "isJsonSchema": true, + "metadata": {}, + "overflow": false, + "plotOptions": { + "customPlotOptions": {}, + "displayType": "table", + "pivotAggregation": null, + "pivotColumns": null, + "xColumns": null, + "yColumns": null + }, + "removedWidgets": [], + "schema": [ + { + "metadata": "{}", + "name": "App ID", + "type": "\"string\"" + }, + { + "metadata": "{}", + "name": "SQL ID", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "Exec Name", + "type": "\"string\"" + }, + { + "metadata": "{}", + "name": "Expression Name", + "type": "\"string\"" + }, + { + "metadata": "{}", + "name": "Task Speedup Factor", + "type": "\"double\"" + }, + { + "metadata": "{}", + "name": "Exec Duration", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "SQL Node Id", + "type": "\"long\"" + }, + { + "metadata": "{}", + "name": "Exec Is Supported", + "type": "\"boolean\"" + }, + { + "metadata": "{}", + "name": "Exec Stages", + "type": "\"string\"" + }, + { + "metadata": "{}", + "name": "Exec Children", + "type": "\"string\"" + }, + { + "metadata": "{}", + "name": "Exec Children Node Ids", + "type": "\"string\"" + }, + { + "metadata": "{}", + "name": "Exec Should Remove", + "type": "\"boolean\"" + } + ], + "type": "table" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "execs_output=pd.read_csv(outputpath_string + \"/rapids_4_spark_qualification_output/rapids_4_spark_qualification_output_execs.csv\")\n", + "display(execs_output)" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "dashboards": [ + { + "elements": [], + "globalVars": {}, + "guid": "", + "layoutOption": { + "grid": true, + "stack": true + }, + "nuid": "91c1bfb2-695a-4e5c-8a25-848a433108dc", + "origId": 556902937009666, + "title": "Executive View", + "version": "DashboardViewV1", + "width": 1600 + }, + { + "elements": [], + "globalVars": {}, + "guid": "", + "layoutOption": { + "grid": true, + "stack": true + }, + "nuid": "62243296-4562-4f06-90ac-d7a609f19c16", + "origId": 556902937009667, + "title": "App View", + "version": "DashboardViewV1", + "width": 1920 + } + ], + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 2, + "widgetLayout": [ + { + "breakBefore": false, + "name": "log_path", + "width": 551 + }, + { + "breakBefore": false, + "name": "output_path", + "width": 551 + } + ] + }, + "notebookName": "[RAPIDS Accelerator for Apache Spark] Qualification User Tool Notebook Template", + "widgets": { + "csp": { + "currentValue": "aws", + "nuid": "c75636ae-f875-4022-98b5-14dbe4ac9957", + "widgetInfo": { + "widgetType": "dropdown", + "defaultValue": "aws", + "label": null, + "name": "csp", + "options": { + "widgetType": "dropdown", + "choices": [ + "aws", + "azure" + ] + } + } + }, + "log_path": { + "currentValue": "/dbfs/mahrens/qualification_logs/app_20220209224316_0007", + "nuid": "88986aa6-6e67-4d09-aeeb-7c96ea1ea8f1", + "widgetInfo": { + "widgetType": "text", + "defaultValue": "", + "label": null, + "name": "log_path", + "options": { + "widgetType": "text", + "validationRegex": null + } + } + }, + "output_path": { + "currentValue": "/tmp", + "nuid": "0b9846bd-5c3e-4ae2-ba00-80bb9da1ee32", + "widgetInfo": { + "widgetType": "text", + "defaultValue": "", + "label": null, + "name": "output_path", + "options": { + "widgetType": "text", + "validationRegex": null + } + } + } + } + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}