From bd93770b7f4190e96919956124a711fb828dc009 Mon Sep 17 00:00:00 2001 From: Ahmed Hussein <50450311+amahussein@users.noreply.github.com> Date: Thu, 15 Feb 2024 13:54:54 -0600 Subject: [PATCH] Qualification should mark empty2null as supported (#791) Signed-off-by: Ahmed Hussein (amahussein) Fixes #787 Add support to expression `Empty2Null` which was added in Spark3.4+ --- core/src/main/resources/operatorsScore-databricks-aws-t4.csv | 1 + core/src/main/resources/operatorsScore-databricks-azure-t4.csv | 1 + core/src/main/resources/operatorsScore-dataproc-gke-l4.csv | 1 + core/src/main/resources/operatorsScore-dataproc-gke-t4.csv | 1 + core/src/main/resources/operatorsScore-dataproc-l4.csv | 1 + .../main/resources/operatorsScore-dataproc-serverless-l4.csv | 1 + core/src/main/resources/operatorsScore-dataproc-t4.csv | 1 + core/src/main/resources/operatorsScore-emr-a10.csv | 1 + core/src/main/resources/operatorsScore-emr-t4.csv | 1 + core/src/main/resources/operatorsScore-onprem-a100.csv | 1 + core/src/main/resources/supportedExprs.csv | 2 ++ 11 files changed, 12 insertions(+) diff --git a/core/src/main/resources/operatorsScore-databricks-aws-t4.csv b/core/src/main/resources/operatorsScore-databricks-aws-t4.csv index 4ce7087e7..a6762fe31 100644 --- a/core/src/main/resources/operatorsScore-databricks-aws-t4.csv +++ b/core/src/main/resources/operatorsScore-databricks-aws-t4.csv @@ -260,6 +260,7 @@ WindowExpression,2.45 WindowSpecDefinition,2.45 XxHash64,2.45 Year,2.45 +Empty2Null,2.45 WriteFilesExec,2.45 AggregateInPandasExec,1.2 ArrowEvalPythonExec,1.2 diff --git a/core/src/main/resources/operatorsScore-databricks-azure-t4.csv b/core/src/main/resources/operatorsScore-databricks-azure-t4.csv index 0c7c65f51..2d5f6bc7d 100644 --- a/core/src/main/resources/operatorsScore-databricks-azure-t4.csv +++ b/core/src/main/resources/operatorsScore-databricks-azure-t4.csv @@ -260,6 +260,7 @@ WindowExpression,2.73 WindowSpecDefinition,2.73 XxHash64,2.73 Year,2.73 +Empty2Null,2.73 WriteFilesExec,2.73 AggregateInPandasExec,1.2 ArrowEvalPythonExec,1.2 diff --git a/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv b/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv index dbbfec22f..dd9985449 100644 --- a/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv @@ -253,6 +253,7 @@ WindowExpression,3.74 WindowSpecDefinition,3.74 XxHash64,3.74 Year,3.74 +Empty2Null,3.74 WriteFilesExec,3.74 AggregateInPandasExec,1.2 ArrowEvalPythonExec,1.2 diff --git a/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv b/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv index cd4bdb55e..3fd8c4661 100644 --- a/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv @@ -253,6 +253,7 @@ WindowExpression,3.65 WindowSpecDefinition,3.65 XxHash64,3.65 Year,3.65 +Empty2Null,3.65 WriteFilesExec,3.65 AggregateInPandasExec,1.2 ArrowEvalPythonExec,1.2 diff --git a/core/src/main/resources/operatorsScore-dataproc-l4.csv b/core/src/main/resources/operatorsScore-dataproc-l4.csv index 9c13792fe..5a5648c14 100644 --- a/core/src/main/resources/operatorsScore-dataproc-l4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-l4.csv @@ -260,6 +260,7 @@ WindowExpression,4.16 WindowSpecDefinition,4.16 XxHash64,4.16 Year,4.16 +Empty2Null,4.16 WriteFilesExec,4.16 AggregateInPandasExec,1.2 ArrowEvalPythonExec,1.2 diff --git a/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv b/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv index 6874994eb..0d3f5f062 100644 --- a/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv @@ -253,6 +253,7 @@ WindowExpression,4.25 WindowSpecDefinition,4.25 XxHash64,4.25 Year,4.25 +Empty2Null,4.25 WriteFilesExec,4.25 AggregateInPandasExec,1.2 ArrowEvalPythonExec,1.2 diff --git a/core/src/main/resources/operatorsScore-dataproc-t4.csv b/core/src/main/resources/operatorsScore-dataproc-t4.csv index d0234e2fd..17cd05fe4 100644 --- a/core/src/main/resources/operatorsScore-dataproc-t4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-t4.csv @@ -260,6 +260,7 @@ WindowExpression,4.88 WindowSpecDefinition,4.88 XxHash64,4.88 Year,4.88 +Empty2Null,4.88 WriteFilesExec,4.88 AggregateInPandasExec,1.2 ArrowEvalPythonExec,1.2 diff --git a/core/src/main/resources/operatorsScore-emr-a10.csv b/core/src/main/resources/operatorsScore-emr-a10.csv index 9c48cc9a4..e83cf0896 100644 --- a/core/src/main/resources/operatorsScore-emr-a10.csv +++ b/core/src/main/resources/operatorsScore-emr-a10.csv @@ -260,6 +260,7 @@ WindowExpression,2.59 WindowSpecDefinition,2.59 XxHash64,2.59 Year,2.59 +Empty2Null,2.59 WriteFilesExec,2.59 AggregateInPandasExec,1.2 ArrowEvalPythonExec,1.2 diff --git a/core/src/main/resources/operatorsScore-emr-t4.csv b/core/src/main/resources/operatorsScore-emr-t4.csv index 503b05994..161996197 100644 --- a/core/src/main/resources/operatorsScore-emr-t4.csv +++ b/core/src/main/resources/operatorsScore-emr-t4.csv @@ -260,6 +260,7 @@ WindowExpression,2.07 WindowSpecDefinition,2.07 XxHash64,2.07 Year,2.07 +Empty2Null,2.07 WriteFilesExec,2.07 AggregateInPandasExec,1.2 ArrowEvalPythonExec,1.2 diff --git a/core/src/main/resources/operatorsScore-onprem-a100.csv b/core/src/main/resources/operatorsScore-onprem-a100.csv index f4367f388..204324162 100644 --- a/core/src/main/resources/operatorsScore-onprem-a100.csv +++ b/core/src/main/resources/operatorsScore-onprem-a100.csv @@ -266,6 +266,7 @@ WindowSpecDefinition,4 XxHash64,4 Year,4 WriteFilesExec,4 +Empty2Null,4 KMeans-pyspark,8.86 KMeans-scala,1 PCA-pyspark,2.24 diff --git a/core/src/main/resources/supportedExprs.csv b/core/src/main/resources/supportedExprs.csv index d1090ca0d..89b3acd27 100644 --- a/core/src/main/resources/supportedExprs.csv +++ b/core/src/main/resources/supportedExprs.csv @@ -728,3 +728,5 @@ HiveGenericUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS HiveGenericUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS HiveSimpleUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS HiveSimpleUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS +Empty2Null,S,`empty2null`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA +Empty2Null,S,`empty2null`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA \ No newline at end of file