Qualification should mark empty2null as supported (#791)

Signed-off-by: Ahmed Hussein (amahussein) <[email protected]> Fixes #787 Add support to expression `Empty2Null` which was added in Spark3.4+
NVIDIA · Feb 15, 2024 · bd93770 · bd93770
1 parent ec5239e
commit bd93770
Show file tree

Hide file tree

Showing 11 changed files with 12 additions and 0 deletions.
diff --git a/core/src/main/resources/operatorsScore-databricks-aws-t4.csv b/core/src/main/resources/operatorsScore-databricks-aws-t4.csv
@@ -260,6 +260,7 @@ WindowExpression,2.45
 WindowSpecDefinition,2.45
 XxHash64,2.45
 Year,2.45
+Empty2Null,2.45
 WriteFilesExec,2.45
 AggregateInPandasExec,1.2
 ArrowEvalPythonExec,1.2

diff --git a/core/src/main/resources/operatorsScore-databricks-azure-t4.csv b/core/src/main/resources/operatorsScore-databricks-azure-t4.csv
@@ -260,6 +260,7 @@ WindowExpression,2.73
 WindowSpecDefinition,2.73
 XxHash64,2.73
 Year,2.73
+Empty2Null,2.73
 WriteFilesExec,2.73
 AggregateInPandasExec,1.2
 ArrowEvalPythonExec,1.2

diff --git a/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv b/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv
@@ -253,6 +253,7 @@ WindowExpression,3.74
 WindowSpecDefinition,3.74
 XxHash64,3.74
 Year,3.74
+Empty2Null,3.74
 WriteFilesExec,3.74
 AggregateInPandasExec,1.2
 ArrowEvalPythonExec,1.2

diff --git a/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv b/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv
@@ -253,6 +253,7 @@ WindowExpression,3.65
 WindowSpecDefinition,3.65
 XxHash64,3.65
 Year,3.65
+Empty2Null,3.65
 WriteFilesExec,3.65
 AggregateInPandasExec,1.2
 ArrowEvalPythonExec,1.2

diff --git a/core/src/main/resources/operatorsScore-dataproc-l4.csv b/core/src/main/resources/operatorsScore-dataproc-l4.csv
@@ -260,6 +260,7 @@ WindowExpression,4.16
 WindowSpecDefinition,4.16
 XxHash64,4.16
 Year,4.16
+Empty2Null,4.16
 WriteFilesExec,4.16
 AggregateInPandasExec,1.2
 ArrowEvalPythonExec,1.2

diff --git a/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv b/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv
@@ -253,6 +253,7 @@ WindowExpression,4.25
 WindowSpecDefinition,4.25
 XxHash64,4.25
 Year,4.25
+Empty2Null,4.25
 WriteFilesExec,4.25
 AggregateInPandasExec,1.2
 ArrowEvalPythonExec,1.2

diff --git a/core/src/main/resources/operatorsScore-dataproc-t4.csv b/core/src/main/resources/operatorsScore-dataproc-t4.csv
@@ -260,6 +260,7 @@ WindowExpression,4.88
 WindowSpecDefinition,4.88
 XxHash64,4.88
 Year,4.88
+Empty2Null,4.88
 WriteFilesExec,4.88
 AggregateInPandasExec,1.2
 ArrowEvalPythonExec,1.2

diff --git a/core/src/main/resources/operatorsScore-emr-a10.csv b/core/src/main/resources/operatorsScore-emr-a10.csv
@@ -260,6 +260,7 @@ WindowExpression,2.59
 WindowSpecDefinition,2.59
 XxHash64,2.59
 Year,2.59
+Empty2Null,2.59
 WriteFilesExec,2.59
 AggregateInPandasExec,1.2
 ArrowEvalPythonExec,1.2

diff --git a/core/src/main/resources/operatorsScore-emr-t4.csv b/core/src/main/resources/operatorsScore-emr-t4.csv
@@ -260,6 +260,7 @@ WindowExpression,2.07
 WindowSpecDefinition,2.07
 XxHash64,2.07
 Year,2.07
+Empty2Null,2.07
 WriteFilesExec,2.07
 AggregateInPandasExec,1.2
 ArrowEvalPythonExec,1.2

diff --git a/core/src/main/resources/operatorsScore-onprem-a100.csv b/core/src/main/resources/operatorsScore-onprem-a100.csv
@@ -266,6 +266,7 @@ WindowSpecDefinition,4
 XxHash64,4
 Year,4
 WriteFilesExec,4
+Empty2Null,4
 KMeans-pyspark,8.86
 KMeans-scala,1
 PCA-pyspark,2.24

diff --git a/core/src/main/resources/supportedExprs.csv b/core/src/main/resources/supportedExprs.csv
@@ -728,3 +728,5 @@ HiveGenericUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS
 HiveGenericUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS
 HiveSimpleUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS
 HiveSimpleUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS
+Empty2Null,S,`empty2null`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA
+Empty2Null,S,`empty2null`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA