From f9076a05c59867de3475e616c3ee46be57e7dc61 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Wed, 22 May 2024 22:56:18 +0800 Subject: [PATCH 01/79] Init version 24.08.0-SNAPSHOT Keep dependencies (JNI + private) as 24.06-SNAPSHOT until they're available. Filed an issue (https://github.com/NVIDIA/spark-rapids/issues/10867) to remind us to bump up dependencies to 24.08.0-SNAPSHOT. Signed-off-by: Tim Liu --- CONTRIBUTING.md | 8 ++++---- README.md | 2 +- aggregator/pom.xml | 4 ++-- api_validation/pom.xml | 4 ++-- datagen/README.md | 6 +++--- datagen/ScaleTest.md | 2 +- datagen/pom.xml | 4 ++-- delta-lake/delta-20x/pom.xml | 4 ++-- delta-lake/delta-21x/pom.xml | 4 ++-- delta-lake/delta-22x/pom.xml | 4 ++-- delta-lake/delta-23x/pom.xml | 4 ++-- delta-lake/delta-24x/pom.xml | 4 ++-- delta-lake/delta-spark330db/pom.xml | 4 ++-- delta-lake/delta-spark332db/pom.xml | 4 ++-- delta-lake/delta-spark341db/pom.xml | 4 ++-- delta-lake/delta-stub/pom.xml | 4 ++-- dist/pom.xml | 4 ++-- docs/configs.md | 2 +- docs/dev/shims.md | 12 ++++++------ docs/dev/testing.md | 4 ++-- integration_tests/README.md | 6 +++--- integration_tests/ScaleTest.md | 2 +- integration_tests/pom.xml | 4 ++-- jdk-profiles/pom.xml | 4 ++-- jenkins/databricks/create.py | 2 +- jenkins/databricks/init_cudf_udf.sh | 2 +- jenkins/version-def.sh | 7 ++++--- pom.xml | 3 ++- scala2.13/aggregator/pom.xml | 4 ++-- scala2.13/api_validation/pom.xml | 4 ++-- scala2.13/datagen/pom.xml | 4 ++-- scala2.13/delta-lake/delta-20x/pom.xml | 4 ++-- scala2.13/delta-lake/delta-21x/pom.xml | 4 ++-- scala2.13/delta-lake/delta-22x/pom.xml | 4 ++-- scala2.13/delta-lake/delta-23x/pom.xml | 4 ++-- scala2.13/delta-lake/delta-24x/pom.xml | 4 ++-- scala2.13/delta-lake/delta-spark330db/pom.xml | 4 ++-- scala2.13/delta-lake/delta-spark332db/pom.xml | 4 ++-- scala2.13/delta-lake/delta-spark341db/pom.xml | 4 ++-- scala2.13/delta-lake/delta-stub/pom.xml | 4 ++-- scala2.13/dist/pom.xml | 4 ++-- scala2.13/integration_tests/pom.xml | 4 ++-- scala2.13/jdk-profiles/pom.xml | 4 ++-- scala2.13/pom.xml | 3 ++- scala2.13/shim-deps/cloudera/pom.xml | 4 ++-- scala2.13/shim-deps/databricks/pom.xml | 4 ++-- scala2.13/shim-deps/pom.xml | 4 ++-- scala2.13/shuffle-plugin/pom.xml | 4 ++-- scala2.13/sql-plugin-api/pom.xml | 4 ++-- scala2.13/sql-plugin/pom.xml | 4 ++-- scala2.13/tests/pom.xml | 4 ++-- scala2.13/tools/pom.xml | 4 ++-- scala2.13/udf-compiler/pom.xml | 4 ++-- shim-deps/cloudera/pom.xml | 4 ++-- shim-deps/databricks/pom.xml | 4 ++-- shim-deps/pom.xml | 4 ++-- shuffle-plugin/pom.xml | 4 ++-- sql-plugin-api/pom.xml | 4 ++-- .../scala/com/nvidia/spark/rapids/ShimLoader.scala | 8 ++++---- sql-plugin/pom.xml | 4 ++-- .../main/scala/com/nvidia/spark/rapids/Plugin.scala | 6 +++--- .../scala/com/nvidia/spark/rapids/RapidsConf.scala | 2 +- tests/pom.xml | 4 ++-- tools/pom.xml | 4 ++-- udf-compiler/pom.xml | 4 ++-- 65 files changed, 136 insertions(+), 133 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index da0e6d8675a..295006be49c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -130,15 +130,15 @@ mvn -pl dist -PnoSnapshots package -DskipTests Verify that shim-specific classes are hidden from a conventional classloader. ```bash -$ javap -cp dist/target/rapids-4-spark_2.12-24.06.0-SNAPSHOT-cuda11.jar com.nvidia.spark.rapids.shims.SparkShimImpl +$ javap -cp dist/target/rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar com.nvidia.spark.rapids.shims.SparkShimImpl Error: class not found: com.nvidia.spark.rapids.shims.SparkShimImpl ``` However, its bytecode can be loaded if prefixed with `spark3XY` not contained in the package name ```bash -$ javap -cp dist/target/rapids-4-spark_2.12-24.06.0-SNAPSHOT-cuda11.jar spark320.com.nvidia.spark.rapids.shims.SparkShimImpl | head -2 -Warning: File dist/target/rapids-4-spark_2.12-24.06.0-SNAPSHOT-cuda11.jar(/spark320/com/nvidia/spark/rapids/shims/SparkShimImpl.class) does not contain class spark320.com.nvidia.spark.rapids.shims.SparkShimImpl +$ javap -cp dist/target/rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar spark320.com.nvidia.spark.rapids.shims.SparkShimImpl | head -2 +Warning: File dist/target/rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar(/spark320/com/nvidia/spark/rapids/shims/SparkShimImpl.class) does not contain class spark320.com.nvidia.spark.rapids.shims.SparkShimImpl Compiled from "SparkShims.scala" public final class com.nvidia.spark.rapids.shims.SparkShimImpl { ``` @@ -181,7 +181,7 @@ mvn package -pl dist -am -Dbuildver=340 -DallowConventionalDistJar=true Verify `com.nvidia.spark.rapids.shims.SparkShimImpl` is conventionally loadable: ```bash -$ javap -cp dist/target/rapids-4-spark_2.12-24.06.0-SNAPSHOT-cuda11.jar com.nvidia.spark.rapids.shims.SparkShimImpl | head -2 +$ javap -cp dist/target/rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar com.nvidia.spark.rapids.shims.SparkShimImpl | head -2 Compiled from "SparkShims.scala" public final class com.nvidia.spark.rapids.shims.SparkShimImpl { ``` diff --git a/README.md b/README.md index 02e5b4cd95a..297e6eacb2f 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ as a `provided` dependency. com.nvidia rapids-4-spark_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT provided ``` diff --git a/aggregator/pom.xml b/aggregator/pom.xml index 7d3d4b94c35..22bfe11105e 100644 --- a/aggregator/pom.xml +++ b/aggregator/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../jdk-profiles/pom.xml rapids-4-spark-aggregator_2.12 RAPIDS Accelerator for Apache Spark Aggregator Creates an aggregated shaded package of the RAPIDS plugin for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT aggregator diff --git a/api_validation/pom.xml b/api_validation/pom.xml index a65993b4e13..7b892754d28 100644 --- a/api_validation/pom.xml +++ b/api_validation/pom.xml @@ -22,11 +22,11 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-api-validation_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT api_validation diff --git a/datagen/README.md b/datagen/README.md index ced310f7c82..0bcd572e65e 100644 --- a/datagen/README.md +++ b/datagen/README.md @@ -24,12 +24,12 @@ Where `$SPARK_VERSION` is a compressed version number, like 330 for Spark 3.3.0. After this the jar should be at `target/datagen_2.12-$PLUGIN_VERSION-spark$SPARK_VERSION.jar` -for example a Spark 3.3.0 jar for the 24.06.0 release would be -`target/datagen_2.12-24.06.0-spark330.jar` +for example a Spark 3.3.0 jar for the 24.08.0 release would be +`target/datagen_2.12-24.08.0-spark330.jar` To get a spark shell with this you can run ```shell -spark-shell --jars target/datagen_2.12-24.06.0-spark330.jar +spark-shell --jars target/datagen_2.12-24.08.0-spark330.jar ``` After that you should be good to go. diff --git a/datagen/ScaleTest.md b/datagen/ScaleTest.md index bdd7c2ff5e7..19ca2d21713 100644 --- a/datagen/ScaleTest.md +++ b/datagen/ScaleTest.md @@ -44,7 +44,7 @@ $SPARK_HOME/bin/spark-submit \ --conf spark.sql.parquet.datetimeRebaseModeInWrite=CORRECTED \ --class com.nvidia.rapids.tests.scaletest.ScaleTestDataGen \ # the main class --jars $SPARK_HOME/examples/jars/scopt_2.12-3.7.1.jar \ # one dependency jar just shipped with Spark under $SPARK_HOME -./target/datagen_2.12-24.06.0-SNAPSHOT-spark332.jar \ +./target/datagen_2.12-24.08.0-SNAPSHOT-spark332.jar \ 1 \ 10 \ parquet \ diff --git a/datagen/pom.xml b/datagen/pom.xml index 9acdfaab044..a18797d049f 100644 --- a/datagen/pom.xml +++ b/datagen/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml datagen_2.12 Data Generator Tools for generating large amounts of data - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT datagen diff --git a/delta-lake/delta-20x/pom.xml b/delta-lake/delta-20x/pom.xml index c180f566cb5..dfc34fad987 100644 --- a/delta-lake/delta-20x/pom.xml +++ b/delta-lake/delta-20x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-20x_2.12 RAPIDS Accelerator for Apache Spark Delta Lake 2.0.x Support Delta Lake 2.0.x support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-20x diff --git a/delta-lake/delta-21x/pom.xml b/delta-lake/delta-21x/pom.xml index 90dcc723f8f..8770a7e3d9d 100644 --- a/delta-lake/delta-21x/pom.xml +++ b/delta-lake/delta-21x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-21x_2.12 RAPIDS Accelerator for Apache Spark Delta Lake 2.1.x Support Delta Lake 2.1.x support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-21x diff --git a/delta-lake/delta-22x/pom.xml b/delta-lake/delta-22x/pom.xml index 82d52e2e896..be9e122e5df 100644 --- a/delta-lake/delta-22x/pom.xml +++ b/delta-lake/delta-22x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-22x_2.12 RAPIDS Accelerator for Apache Spark Delta Lake 2.2.x Support Delta Lake 2.2.x support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-22x diff --git a/delta-lake/delta-23x/pom.xml b/delta-lake/delta-23x/pom.xml index bf003810e6f..97207dca741 100644 --- a/delta-lake/delta-23x/pom.xml +++ b/delta-lake/delta-23x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-parent_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../pom.xml rapids-4-spark-delta-23x_2.12 RAPIDS Accelerator for Apache Spark Delta Lake 2.3.x Support Delta Lake 2.3.x support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-23x diff --git a/delta-lake/delta-24x/pom.xml b/delta-lake/delta-24x/pom.xml index 1cfcbaa8385..19e0be3c90d 100644 --- a/delta-lake/delta-24x/pom.xml +++ b/delta-lake/delta-24x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-24x_2.12 RAPIDS Accelerator for Apache Spark Delta Lake 2.4.x Support Delta Lake 2.4.x support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-24x diff --git a/delta-lake/delta-spark330db/pom.xml b/delta-lake/delta-spark330db/pom.xml index d33d4b16d67..abc57f793f8 100644 --- a/delta-lake/delta-spark330db/pom.xml +++ b/delta-lake/delta-spark330db/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-spark330db_2.12 RAPIDS Accelerator for Apache Spark Databricks 11.3 Delta Lake Support Databricks 11.3 Delta Lake support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-spark330db diff --git a/delta-lake/delta-spark332db/pom.xml b/delta-lake/delta-spark332db/pom.xml index 7aaf04dbc7b..4511a76be71 100644 --- a/delta-lake/delta-spark332db/pom.xml +++ b/delta-lake/delta-spark332db/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-spark332db_2.12 RAPIDS Accelerator for Apache Spark Databricks 12.2 Delta Lake Support Databricks 12.2 Delta Lake support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-spark332db diff --git a/delta-lake/delta-spark341db/pom.xml b/delta-lake/delta-spark341db/pom.xml index d4f9ad9c436..6532062af68 100644 --- a/delta-lake/delta-spark341db/pom.xml +++ b/delta-lake/delta-spark341db/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-spark341db_2.12 RAPIDS Accelerator for Apache Spark Databricks 13.3 Delta Lake Support Databricks 13.3 Delta Lake support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT false diff --git a/delta-lake/delta-stub/pom.xml b/delta-lake/delta-stub/pom.xml index 28c6db7a251..bfe920e3dc4 100644 --- a/delta-lake/delta-stub/pom.xml +++ b/delta-lake/delta-stub/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-stub_2.12 RAPIDS Accelerator for Apache Spark Delta Lake Stub Delta Lake stub for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-stub diff --git a/dist/pom.xml b/dist/pom.xml index f2f9ea7e17a..ecc2018baea 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../jdk-profiles/pom.xml rapids-4-spark_2.12 RAPIDS Accelerator for Apache Spark Distribution Creates the distribution package of the RAPIDS plugin for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT com.nvidia diff --git a/docs/configs.md b/docs/configs.md index 57517bc2b32..def85c8ac97 100644 --- a/docs/configs.md +++ b/docs/configs.md @@ -10,7 +10,7 @@ The following is the list of options that `rapids-plugin-4-spark` supports. On startup use: `--conf [conf key]=[conf value]`. For example: ``` -${SPARK_HOME}/bin/spark-shell --jars rapids-4-spark_2.12-24.06.0-SNAPSHOT-cuda11.jar \ +${SPARK_HOME}/bin/spark-shell --jars rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar \ --conf spark.plugins=com.nvidia.spark.SQLPlugin \ --conf spark.rapids.sql.concurrentGpuTasks=2 ``` diff --git a/docs/dev/shims.md b/docs/dev/shims.md index 9a8e09d8295..b6290287314 100644 --- a/docs/dev/shims.md +++ b/docs/dev/shims.md @@ -68,17 +68,17 @@ Using JarURLConnection URLs we create a Parallel World of the current version wi Spark 3.0.2's URLs: ```text -jar:file:/home/spark/rapids-4-spark_2.12-24.06.0.jar!/ -jar:file:/home/spark/rapids-4-spark_2.12-24.06.0.jar!/spark3xx-common/ -jar:file:/home/spark/rapids-4-spark_2.12-24.06.0.jar!/spark302/ +jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/ +jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark3xx-common/ +jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark302/ ``` Spark 3.2.0's URLs : ```text -jar:file:/home/spark/rapids-4-spark_2.12-24.06.0.jar!/ -jar:file:/home/spark/rapids-4-spark_2.12-24.06.0.jar!/spark3xx-common/ -jar:file:/home/spark/rapids-4-spark_2.12-24.06.0.jar!/spark320/ +jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/ +jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark3xx-common/ +jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark320/ ``` ### Late Inheritance in Public Classes diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 6a6c6a378eb..144f11090b8 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -5,5 +5,5 @@ nav_order: 2 parent: Developer Overview --- An overview of testing can be found within the repository at: -* [Unit tests](https://github.com/NVIDIA/spark-rapids/tree/branch-24.06/tests#readme) -* [Integration testing](https://github.com/NVIDIA/spark-rapids/tree/branch-24.06/integration_tests#readme) +* [Unit tests](https://github.com/NVIDIA/spark-rapids/tree/branch-24.08/tests#readme) +* [Integration testing](https://github.com/NVIDIA/spark-rapids/tree/branch-24.08/integration_tests#readme) diff --git a/integration_tests/README.md b/integration_tests/README.md index 9493fbb07d1..c5f55f9f605 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -263,7 +263,7 @@ individually, so you don't risk running unit tests along with the integration te http://www.scalatest.org/user_guide/using_the_scalatest_shell ```shell -spark-shell --jars rapids-4-spark-tests_2.12-24.06.0-SNAPSHOT-tests.jar,rapids-4-spark-integration-tests_2.12-24.06.0-SNAPSHOT-tests.jar,scalatest_2.12-3.0.5.jar,scalactic_2.12-3.0.5.jar +spark-shell --jars rapids-4-spark-tests_2.12-24.08.0-SNAPSHOT-tests.jar,rapids-4-spark-integration-tests_2.12-24.08.0-SNAPSHOT-tests.jar,scalatest_2.12-3.0.5.jar,scalactic_2.12-3.0.5.jar ``` First you import the `scalatest_shell` and tell the tests where they can find the test files you @@ -286,7 +286,7 @@ If you just want to verify the SQL replacement is working you will need to add t assumes CUDA 11.0 is being used and the Spark distribution is built with Scala 2.12. ``` -$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-24.06.0-SNAPSHOT-cuda11.jar" ./runtests.py +$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar" ./runtests.py ``` You don't have to enable the plugin for this to work, the test framework will do that for you. @@ -443,7 +443,7 @@ To run cudf_udf tests, need following configuration changes: As an example, here is the `spark-submit` command with the cudf_udf parameter on CUDA 11.0: ``` -$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-24.06.0-SNAPSHOT-cuda11.jar,rapids-4-spark-tests_2.12-24.06.0-SNAPSHOT.jar" --conf spark.rapids.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.concurrentPythonWorkers=2 --py-files "rapids-4-spark_2.12-24.06.0-SNAPSHOT-cuda11.jar" --conf spark.executorEnv.PYTHONPATH="rapids-4-spark_2.12-24.06.0-SNAPSHOT-cuda11.jar" ./runtests.py --cudf_udf +$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar,rapids-4-spark-tests_2.12-24.08.0-SNAPSHOT.jar" --conf spark.rapids.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.concurrentPythonWorkers=2 --py-files "rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar" --conf spark.executorEnv.PYTHONPATH="rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar" ./runtests.py --cudf_udf ``` ### Enabling fuzz tests diff --git a/integration_tests/ScaleTest.md b/integration_tests/ScaleTest.md index cd99b7cc7e9..a8f7fb991f3 100644 --- a/integration_tests/ScaleTest.md +++ b/integration_tests/ScaleTest.md @@ -97,7 +97,7 @@ $SPARK_HOME/bin/spark-submit \ --conf spark.sql.parquet.datetimeRebaseModeInWrite=CORRECTED \ --jars $SPARK_HOME/examples/jars/scopt_2.12-3.7.1.jar \ --class com.nvidia.spark.rapids.tests.scaletest.ScaleTest \ -./target/rapids-4-spark-integration-tests_2.12-24.06.0-SNAPSHOT-spark332.jar \ +./target/rapids-4-spark-integration-tests_2.12-24.08.0-SNAPSHOT-spark332.jar \ 10 \ 100 \ parquet \ diff --git a/integration_tests/pom.xml b/integration_tests/pom.xml index f413d24b70a..75bd244f31d 100644 --- a/integration_tests/pom.xml +++ b/integration_tests/pom.xml @@ -22,11 +22,11 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-integration-tests_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT integration_tests diff --git a/jdk-profiles/pom.xml b/jdk-profiles/pom.xml index ad865205946..771b5a72476 100644 --- a/jdk-profiles/pom.xml +++ b/jdk-profiles/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT com.nvidia rapids-4-spark-jdk-profiles_2.12 pom Shim JDK Profiles - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT jdk9plus diff --git a/jenkins/databricks/create.py b/jenkins/databricks/create.py index 4da25a328b8..58b583b5c08 100644 --- a/jenkins/databricks/create.py +++ b/jenkins/databricks/create.py @@ -27,7 +27,7 @@ def main(): workspace = 'https://dbc-9ff9942e-a9c4.cloud.databricks.com' token = '' sshkey = '' - cluster_name = 'CI-GPU-databricks-24.06.0-SNAPSHOT' + cluster_name = 'CI-GPU-databricks-24.08.0-SNAPSHOT' idletime = 240 runtime = '7.0.x-gpu-ml-scala2.12' num_workers = 1 diff --git a/jenkins/databricks/init_cudf_udf.sh b/jenkins/databricks/init_cudf_udf.sh index f214c7d27e9..d5c440bfbb2 100755 --- a/jenkins/databricks/init_cudf_udf.sh +++ b/jenkins/databricks/init_cudf_udf.sh @@ -20,7 +20,7 @@ set -ex -CUDF_VER=${CUDF_VER:-24.06} +CUDF_VER=${CUDF_VER:-24.06} # TODO: https://github.com/NVIDIA/spark-rapids/issues/ CUDA_VER=${CUDA_VER:-11.8} # Need to explicitly add conda into PATH environment, to activate conda environment. diff --git a/jenkins/version-def.sh b/jenkins/version-def.sh index 45f46940c37..d3c01e1eba4 100755 --- a/jenkins/version-def.sh +++ b/jenkins/version-def.sh @@ -26,11 +26,12 @@ for VAR in $OVERWRITE_PARAMS; do done IFS=$PRE_IFS -CUDF_VER=${CUDF_VER:-"24.06.0-SNAPSHOT"} + +CUDF_VER=${CUDF_VER:-"24.06.0-SNAPSHOT"} # TODO: https://github.com/NVIDIA/spark-rapids/issues/ CUDA_CLASSIFIER=${CUDA_CLASSIFIER:-"cuda11"} CLASSIFIER=${CLASSIFIER:-"$CUDA_CLASSIFIER"} # default as CUDA_CLASSIFIER for compatibility -PROJECT_VER=${PROJECT_VER:-"24.06.0-SNAPSHOT"} -PROJECT_TEST_VER=${PROJECT_TEST_VER:-"24.06.0-SNAPSHOT"} +PROJECT_VER=${PROJECT_VER:-"24.08.0-SNAPSHOT"} +PROJECT_TEST_VER=${PROJECT_TEST_VER:-"24.08.0-SNAPSHOT"} SPARK_VER=${SPARK_VER:-"3.1.1"} SPARK_VER_213=${SPARK_VER_213:-"3.3.0"} # Make a best attempt to set the default value for the shuffle shim. diff --git a/pom.xml b/pom.xml index c939f8d5891..df010a7589e 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ rapids-4-spark-parent_2.12 RAPIDS Accelerator for Apache Spark Root Project The root project of the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT pom https://nvidia.github.io/spark-rapids/ @@ -719,6 +719,7 @@ spark${buildver} cuda11 ${cuda.version} + 24.06.0-SNAPSHOT 24.06.0-SNAPSHOT 2.12 diff --git a/scala2.13/aggregator/pom.xml b/scala2.13/aggregator/pom.xml index f3bbfc1d7dc..1d70c76f037 100644 --- a/scala2.13/aggregator/pom.xml +++ b/scala2.13/aggregator/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../jdk-profiles/pom.xml rapids-4-spark-aggregator_2.13 RAPIDS Accelerator for Apache Spark Aggregator Creates an aggregated shaded package of the RAPIDS plugin for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT aggregator diff --git a/scala2.13/api_validation/pom.xml b/scala2.13/api_validation/pom.xml index c934b610e73..5cf34d9acf7 100644 --- a/scala2.13/api_validation/pom.xml +++ b/scala2.13/api_validation/pom.xml @@ -22,11 +22,11 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-api-validation_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT api_validation diff --git a/scala2.13/datagen/pom.xml b/scala2.13/datagen/pom.xml index 65c0e4cc98b..c0195411227 100644 --- a/scala2.13/datagen/pom.xml +++ b/scala2.13/datagen/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml datagen_2.13 Data Generator Tools for generating large amounts of data - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT datagen diff --git a/scala2.13/delta-lake/delta-20x/pom.xml b/scala2.13/delta-lake/delta-20x/pom.xml index 9d86042c395..6b098f9bdb1 100644 --- a/scala2.13/delta-lake/delta-20x/pom.xml +++ b/scala2.13/delta-lake/delta-20x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-20x_2.13 RAPIDS Accelerator for Apache Spark Delta Lake 2.0.x Support Delta Lake 2.0.x support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-20x diff --git a/scala2.13/delta-lake/delta-21x/pom.xml b/scala2.13/delta-lake/delta-21x/pom.xml index 83bf7f1f56d..6f58d97b9c9 100644 --- a/scala2.13/delta-lake/delta-21x/pom.xml +++ b/scala2.13/delta-lake/delta-21x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-21x_2.13 RAPIDS Accelerator for Apache Spark Delta Lake 2.1.x Support Delta Lake 2.1.x support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-21x diff --git a/scala2.13/delta-lake/delta-22x/pom.xml b/scala2.13/delta-lake/delta-22x/pom.xml index c0cf82529ad..500405c9f1a 100644 --- a/scala2.13/delta-lake/delta-22x/pom.xml +++ b/scala2.13/delta-lake/delta-22x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-22x_2.13 RAPIDS Accelerator for Apache Spark Delta Lake 2.2.x Support Delta Lake 2.2.x support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-22x diff --git a/scala2.13/delta-lake/delta-23x/pom.xml b/scala2.13/delta-lake/delta-23x/pom.xml index 8eb2a09375f..c6e9cf634a6 100644 --- a/scala2.13/delta-lake/delta-23x/pom.xml +++ b/scala2.13/delta-lake/delta-23x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-parent_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../pom.xml rapids-4-spark-delta-23x_2.13 RAPIDS Accelerator for Apache Spark Delta Lake 2.3.x Support Delta Lake 2.3.x support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-23x diff --git a/scala2.13/delta-lake/delta-24x/pom.xml b/scala2.13/delta-lake/delta-24x/pom.xml index 505e2eb04d7..987d24b0172 100644 --- a/scala2.13/delta-lake/delta-24x/pom.xml +++ b/scala2.13/delta-lake/delta-24x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-24x_2.13 RAPIDS Accelerator for Apache Spark Delta Lake 2.4.x Support Delta Lake 2.4.x support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-24x diff --git a/scala2.13/delta-lake/delta-spark330db/pom.xml b/scala2.13/delta-lake/delta-spark330db/pom.xml index db71dd83111..b5f5a9e475b 100644 --- a/scala2.13/delta-lake/delta-spark330db/pom.xml +++ b/scala2.13/delta-lake/delta-spark330db/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-spark330db_2.13 RAPIDS Accelerator for Apache Spark Databricks 11.3 Delta Lake Support Databricks 11.3 Delta Lake support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-spark330db diff --git a/scala2.13/delta-lake/delta-spark332db/pom.xml b/scala2.13/delta-lake/delta-spark332db/pom.xml index ab9ac45d775..2ea75281f26 100644 --- a/scala2.13/delta-lake/delta-spark332db/pom.xml +++ b/scala2.13/delta-lake/delta-spark332db/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-spark332db_2.13 RAPIDS Accelerator for Apache Spark Databricks 12.2 Delta Lake Support Databricks 12.2 Delta Lake support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-spark332db diff --git a/scala2.13/delta-lake/delta-spark341db/pom.xml b/scala2.13/delta-lake/delta-spark341db/pom.xml index 78be4fad4b8..30c513c6f3d 100644 --- a/scala2.13/delta-lake/delta-spark341db/pom.xml +++ b/scala2.13/delta-lake/delta-spark341db/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-spark341db_2.13 RAPIDS Accelerator for Apache Spark Databricks 13.3 Delta Lake Support Databricks 13.3 Delta Lake support for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT false diff --git a/scala2.13/delta-lake/delta-stub/pom.xml b/scala2.13/delta-lake/delta-stub/pom.xml index 81731f59c4f..2ece8c825f6 100644 --- a/scala2.13/delta-lake/delta-stub/pom.xml +++ b/scala2.13/delta-lake/delta-stub/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../jdk-profiles/pom.xml rapids-4-spark-delta-stub_2.13 RAPIDS Accelerator for Apache Spark Delta Lake Stub Delta Lake stub for the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../delta-lake/delta-stub diff --git a/scala2.13/dist/pom.xml b/scala2.13/dist/pom.xml index 0fd9d2795b6..db9ada51a28 100644 --- a/scala2.13/dist/pom.xml +++ b/scala2.13/dist/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../jdk-profiles/pom.xml rapids-4-spark_2.13 RAPIDS Accelerator for Apache Spark Distribution Creates the distribution package of the RAPIDS plugin for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT com.nvidia diff --git a/scala2.13/integration_tests/pom.xml b/scala2.13/integration_tests/pom.xml index 8dfeda2eeb6..f7e443d5a22 100644 --- a/scala2.13/integration_tests/pom.xml +++ b/scala2.13/integration_tests/pom.xml @@ -22,11 +22,11 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-integration-tests_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT integration_tests diff --git a/scala2.13/jdk-profiles/pom.xml b/scala2.13/jdk-profiles/pom.xml index c41c5ef38f4..23fee358668 100644 --- a/scala2.13/jdk-profiles/pom.xml +++ b/scala2.13/jdk-profiles/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT com.nvidia rapids-4-spark-jdk-profiles_2.13 pom Shim JDK Profiles - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT jdk9plus diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index cf3bfb48373..711872e8d54 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -23,7 +23,7 @@ rapids-4-spark-parent_2.13 RAPIDS Accelerator for Apache Spark Root Project The root project of the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT pom https://nvidia.github.io/spark-rapids/ @@ -719,6 +719,7 @@ spark${buildver} cuda11 ${cuda.version} + 24.06.0-SNAPSHOT 24.06.0-SNAPSHOT 2.13 diff --git a/scala2.13/shim-deps/cloudera/pom.xml b/scala2.13/shim-deps/cloudera/pom.xml index bce1e909069..c238ca81af0 100644 --- a/scala2.13/shim-deps/cloudera/pom.xml +++ b/scala2.13/shim-deps/cloudera/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../pom.xml rapids-4-spark-cdh-bom pom CDH Shim Dependencies - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/cloudera diff --git a/scala2.13/shim-deps/databricks/pom.xml b/scala2.13/shim-deps/databricks/pom.xml index 8b8e9403c30..b342f381c71 100644 --- a/scala2.13/shim-deps/databricks/pom.xml +++ b/scala2.13/shim-deps/databricks/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../pom.xml rapids-4-spark-db-bom pom Databricks Shim Dependencies - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/databricks diff --git a/scala2.13/shim-deps/pom.xml b/scala2.13/shim-deps/pom.xml index 6ab6f90130e..2db84e7bc31 100644 --- a/scala2.13/shim-deps/pom.xml +++ b/scala2.13/shim-deps/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../jdk-profiles/pom.xml rapids-4-spark-shim-deps-parent_2.13 pom Shim Dependencies Profiles - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT release321cdh diff --git a/scala2.13/shuffle-plugin/pom.xml b/scala2.13/shuffle-plugin/pom.xml index a3ee1ffb638..61e69784972 100644 --- a/scala2.13/shuffle-plugin/pom.xml +++ b/scala2.13/shuffle-plugin/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-shuffle_2.13 RAPIDS Accelerator for Apache Spark Shuffle Plugin Accelerated shuffle plugin for the RAPIDS plugin for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT shuffle-plugin diff --git a/scala2.13/sql-plugin-api/pom.xml b/scala2.13/sql-plugin-api/pom.xml index 22736968bf4..e56caf01867 100644 --- a/scala2.13/sql-plugin-api/pom.xml +++ b/scala2.13/sql-plugin-api/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-sql-plugin-api_2.13 Module for Non-Shimmable API - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT sql-plugin-api false diff --git a/scala2.13/sql-plugin/pom.xml b/scala2.13/sql-plugin/pom.xml index 5bee34752c7..df3532a3592 100644 --- a/scala2.13/sql-plugin/pom.xml +++ b/scala2.13/sql-plugin/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-sql_2.13 RAPIDS Accelerator for Apache Spark SQL Plugin The RAPIDS SQL plugin for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT sql-plugin diff --git a/scala2.13/tests/pom.xml b/scala2.13/tests/pom.xml index 59668c4a7fc..4a80f1d3942 100644 --- a/scala2.13/tests/pom.xml +++ b/scala2.13/tests/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-tests_2.13 RAPIDS Accelerator for Apache Spark Tests RAPIDS plugin for Apache Spark integration tests - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT tests diff --git a/scala2.13/tools/pom.xml b/scala2.13/tools/pom.xml index 43796a41532..fd14e27c935 100644 --- a/scala2.13/tools/pom.xml +++ b/scala2.13/tools/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../jdk-profiles/pom.xml rapids-4-spark-tools-support pom RAPIDS Accelerator for Apache Spark Tools Support Supporting code for RAPIDS Accelerator tools - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT com.nvidia diff --git a/scala2.13/udf-compiler/pom.xml b/scala2.13/udf-compiler/pom.xml index 89d96dd41fd..ea16fb9a7c9 100644 --- a/scala2.13/udf-compiler/pom.xml +++ b/scala2.13/udf-compiler/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-udf_2.13 RAPIDS Accelerator for Apache Spark Scala UDF Plugin The RAPIDS Scala UDF plugin for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT udf-compiler diff --git a/shim-deps/cloudera/pom.xml b/shim-deps/cloudera/pom.xml index 787975be559..e8c66f54755 100644 --- a/shim-deps/cloudera/pom.xml +++ b/shim-deps/cloudera/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../pom.xml rapids-4-spark-cdh-bom pom CDH Shim Dependencies - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/cloudera diff --git a/shim-deps/databricks/pom.xml b/shim-deps/databricks/pom.xml index 189faf97a60..bef8a90d227 100644 --- a/shim-deps/databricks/pom.xml +++ b/shim-deps/databricks/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../../pom.xml rapids-4-spark-db-bom pom Databricks Shim Dependencies - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/databricks diff --git a/shim-deps/pom.xml b/shim-deps/pom.xml index 2859bbff28d..5bff137efea 100644 --- a/shim-deps/pom.xml +++ b/shim-deps/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../jdk-profiles/pom.xml rapids-4-spark-shim-deps-parent_2.12 pom Shim Dependencies Profiles - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT release321cdh diff --git a/shuffle-plugin/pom.xml b/shuffle-plugin/pom.xml index 60d7580c237..d3498f8b44f 100644 --- a/shuffle-plugin/pom.xml +++ b/shuffle-plugin/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-shuffle_2.12 RAPIDS Accelerator for Apache Spark Shuffle Plugin Accelerated shuffle plugin for the RAPIDS plugin for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT shuffle-plugin diff --git a/sql-plugin-api/pom.xml b/sql-plugin-api/pom.xml index 68e20c10f77..5e933899560 100644 --- a/sql-plugin-api/pom.xml +++ b/sql-plugin-api/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-sql-plugin-api_2.12 Module for Non-Shimmable API - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT sql-plugin-api false diff --git a/sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala b/sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala index cdcb3f73423..36abc75ba87 100644 --- a/sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala +++ b/sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala @@ -49,11 +49,11 @@ import org.apache.spark.util.MutableURLClassLoader Each shim can see a consistent parallel world without conflicts by referencing only one conflicting directory. E.g., Spark 3.2.0 Shim will use - jar:file:/home/spark/rapids-4-spark_2.12-24.06.0.jar!/spark3xx-common/ - jar:file:/home/spark/rapids-4-spark_2.12-24.06.0.jar!/spark320/ + jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark3xx-common/ + jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark320/ Spark 3.1.1 will use - jar:file:/home/spark/rapids-4-spark_2.12-24.06.0.jar!/spark3xx-common/ - jar:file:/home/spark/rapids-4-spark_2.12-24.06.0.jar!/spark311/ + jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark3xx-common/ + jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark311/ Using these Jar URL's allows referencing different bytecode produced from identical sources by incompatible Scala / Spark dependencies. */ diff --git a/sql-plugin/pom.xml b/sql-plugin/pom.xml index cbe7d873dff..961e6f08372 100644 --- a/sql-plugin/pom.xml +++ b/sql-plugin/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-sql_2.12 RAPIDS Accelerator for Apache Spark SQL Plugin The RAPIDS SQL plugin for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT sql-plugin diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/Plugin.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/Plugin.scala index bd5d76e2337..55df9e5cea6 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/Plugin.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/Plugin.scala @@ -134,11 +134,11 @@ object RapidsPluginUtils extends Logging { val possibleRapidsJarURLs = classloader.getResources(propName).asScala.toSet.toSeq.filter { url => { val urlPath = url.toString - // Filter out submodule jars, e.g. rapids-4-spark-aggregator_2.12-24.06.0-spark341.jar, + // Filter out submodule jars, e.g. rapids-4-spark-aggregator_2.12-24.08.0-spark341.jar, // and files stored under subdirs of '!/', e.g. - // rapids-4-spark_2.12-24.06.0-cuda11.jar!/spark330/rapids4spark-version-info.properties + // rapids-4-spark_2.12-24.08.0-cuda11.jar!/spark330/rapids4spark-version-info.properties // We only want to find the main jar, e.g. - // rapids-4-spark_2.12-24.06.0-cuda11.jar!/rapids4spark-version-info.properties + // rapids-4-spark_2.12-24.08.0-cuda11.jar!/rapids4spark-version-info.properties !urlPath.contains("rapids-4-spark-") && urlPath.endsWith("!/" + propName) } } diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index 6dd0441f76f..befece6d439 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -2237,7 +2237,7 @@ val SHUFFLE_COMPRESSION_LZ4_CHUNK_SIZE = conf("spark.rapids.shuffle.compression. |On startup use: `--conf [conf key]=[conf value]`. For example: | |``` - |${SPARK_HOME}/bin/spark-shell --jars rapids-4-spark_2.12-24.06.0-SNAPSHOT-cuda11.jar \ + |${SPARK_HOME}/bin/spark-shell --jars rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar \ |--conf spark.plugins=com.nvidia.spark.SQLPlugin \ |--conf spark.rapids.sql.concurrentGpuTasks=2 |``` diff --git a/tests/pom.xml b/tests/pom.xml index f8a7067363d..cff108e3253 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-tests_2.12 RAPIDS Accelerator for Apache Spark Tests RAPIDS plugin for Apache Spark integration tests - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT tests diff --git a/tools/pom.xml b/tools/pom.xml index 8b9dabcc791..5fa7391d34c 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../jdk-profiles/pom.xml rapids-4-spark-tools-support pom RAPIDS Accelerator for Apache Spark Tools Support Supporting code for RAPIDS Accelerator tools - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT com.nvidia diff --git a/udf-compiler/pom.xml b/udf-compiler/pom.xml index 9bbd21353c5..edbdf2148cc 100644 --- a/udf-compiler/pom.xml +++ b/udf-compiler/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT ../shim-deps/pom.xml rapids-4-spark-udf_2.12 RAPIDS Accelerator for Apache Spark Scala UDF Plugin The RAPIDS Scala UDF plugin for Apache Spark - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT udf-compiler From 02f45950011da0906f030e840874314de12fe188 Mon Sep 17 00:00:00 2001 From: Zach Puller Date: Tue, 28 May 2024 13:29:47 -0700 Subject: [PATCH 02/79] append zpuller to authorized user of blossom-ci (#10929) Signed-off-by: Zach Puller --- .github/workflows/blossom-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml index 848be4e0a56..b3cbbb6ad14 100644 --- a/.github/workflows/blossom-ci.yml +++ b/.github/workflows/blossom-ci.yml @@ -70,6 +70,7 @@ jobs: parthosa,\ liurenjie1024,\ binmahone,\ + zpuller,\ ', format('{0},', github.actor)) && github.event.comment.body == 'build' steps: - name: Check if comment is issued by authorized person From f0b13ed451086bda4b786187c800fc3afd9ea9f5 Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Wed, 29 May 2024 08:20:48 -0700 Subject: [PATCH 03/79] Fixed Databricks build [databricks] (#10933) * Fixed Databricks build * Signing off Signed-off-by: Raza Jafri * Removed unused import --------- Signed-off-by: Raza Jafri --- .../nvidia/spark/rapids/shims/Spark321PlusDBShims.scala | 4 ++-- .../execution/{shims => }/GpuSubqueryBroadcastMeta.scala | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) rename sql-plugin/src/main/spark330db/scala/org/apache/spark/rapids/execution/{shims => }/GpuSubqueryBroadcastMeta.scala (96%) diff --git a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/Spark321PlusDBShims.scala b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/Spark321PlusDBShims.scala index 5e5bbed2942..fb60f15db4d 100644 --- a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/Spark321PlusDBShims.scala +++ b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/Spark321PlusDBShims.scala @@ -41,8 +41,8 @@ import org.apache.spark.sql.execution.exchange._ import org.apache.spark.sql.execution.python._ import org.apache.spark.sql.execution.window._ import org.apache.spark.sql.rapids.GpuSubstring -import org.apache.spark.sql.rapids.execution._ -import org.apache.spark.sql.rapids.execution.shims.{GpuSubqueryBroadcastMeta,ReuseGpuBroadcastExchangeAndSubquery} +import org.apache.spark.sql.rapids.execution.GpuSubqueryBroadcastMeta +import org.apache.spark.sql.rapids.execution.shims.ReuseGpuBroadcastExchangeAndSubquery import org.apache.spark.sql.rapids.shims._ import org.apache.spark.sql.types._ diff --git a/sql-plugin/src/main/spark330db/scala/org/apache/spark/rapids/execution/shims/GpuSubqueryBroadcastMeta.scala b/sql-plugin/src/main/spark330db/scala/org/apache/spark/rapids/execution/GpuSubqueryBroadcastMeta.scala similarity index 96% rename from sql-plugin/src/main/spark330db/scala/org/apache/spark/rapids/execution/shims/GpuSubqueryBroadcastMeta.scala rename to sql-plugin/src/main/spark330db/scala/org/apache/spark/rapids/execution/GpuSubqueryBroadcastMeta.scala index fbc013779c4..76255b3e5a6 100644 --- a/sql-plugin/src/main/spark330db/scala/org/apache/spark/rapids/execution/shims/GpuSubqueryBroadcastMeta.scala +++ b/sql-plugin/src/main/spark330db/scala/org/apache/spark/rapids/execution/GpuSubqueryBroadcastMeta.scala @@ -19,7 +19,7 @@ {"spark": "332db"} {"spark": "341db"} spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.rapids.execution.shims +package org.apache.spark.sql.rapids.execution import com.nvidia.spark.rapids.{BaseExprMeta, DataFromReplacementRule, GpuExec, RapidsConf, RapidsMeta, SparkPlanMeta} @@ -29,7 +29,6 @@ import org.apache.spark.sql.execution.{SparkPlan, SubqueryBroadcastExec} import org.apache.spark.sql.execution.adaptive.{BroadcastQueryStageExec} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec} import org.apache.spark.sql.execution.joins.HashedRelationBroadcastMode -import org.apache.spark.sql.rapids.execution._ class GpuSubqueryBroadcastMeta( s: SubqueryBroadcastExec, @@ -94,7 +93,7 @@ class GpuSubqueryBroadcastMeta( override def convertToCpu(): SparkPlan = s override def convertToGpu(): GpuExec = { - GpuSubqueryBroadcastExec(s.name, s.index, s.buildKeys, broadcastBuilder())( + GpuSubqueryBroadcastExec(s.name, Seq(s.index), s.buildKeys, broadcastBuilder())( getBroadcastModeKeyExprs) } @@ -123,4 +122,4 @@ class GpuSubqueryBroadcastMeta( case m => throw new UnsupportedOperationException(s"Unknown broadcast mode $m") } } -} \ No newline at end of file +} From dfcde727a9c744a008e256c25d202336639b2e4c Mon Sep 17 00:00:00 2001 From: Zach Puller Date: Wed, 29 May 2024 09:17:51 -0700 Subject: [PATCH 04/79] Add classloader diagnostics to initShuffleManager error message (#10871) Add classloader diagnostics to initShuffleManager error message --------- Signed-off-by: Zach Puller Co-authored-by: Jason Lowe Co-authored-by: Gera Shegalov Co-authored-by: Alessandro Bellina --- .../apache/spark/sql/rapids/GpuShuffleEnv.scala | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuShuffleEnv.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuShuffleEnv.scala index 1682dd13c22..1b0ee21d494 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuShuffleEnv.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuShuffleEnv.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -104,11 +104,22 @@ object GpuShuffleEnv extends Logging { // this forces the initialization when we know we are ready in the driver and executor. // def initShuffleManager(): Unit = { - SparkEnv.get.shuffleManager match { + val shuffleManager = SparkEnv.get.shuffleManager + shuffleManager match { case rapidsShuffleManager: RapidsShuffleManagerLike => rapidsShuffleManager.initialize case _ => - throw new IllegalStateException(s"Cannot initialize the RAPIDS Shuffle Manager") + val rsmLoaderViaShuffleManager = shuffleManager.getClass.getSuperclass.getInterfaces + .collectFirst { + case c if c.getName == classOf[RapidsShuffleManagerLike].getName => c.getClassLoader + } + val rsmLoaderDirect = classOf[RapidsShuffleManagerLike].getClassLoader + + throw new IllegalStateException(s"Cannot initialize the RAPIDS Shuffle Manager " + + s"${shuffleManager}! Expected: an instance of RapidsShuffleManagerLike loaded by " + + s"${rsmLoaderDirect}. Actual: ${shuffleManager} tagged with RapidsShuffleManagerLike " + + s"loaded by: ${rsmLoaderViaShuffleManager}" + ) } } From fe69470fb07281c77383be6dd02a45c39ea5d7e8 Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Wed, 29 May 2024 22:57:17 -0700 Subject: [PATCH 05/79] Add Support for Multiple Filtering Keys for Subquery Broadcast [databricks] (#10945) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revert "Revert "Add Support for Multiple Filtering Keys for Subquery Broadcas…" This reverts commit bb05b179acc7c4a17bbf794d16ce3f0273e0f9dc. * Signing off Signed-off-by: Raza Jafri --------- Signed-off-by: Raza Jafri --- .../execution/GpuSubqueryBroadcastExec.scala | 63 +++++++++---------- .../execution/GpuSubqueryBroadcastMeta.scala | 56 +++++++++++++++++ .../execution/GpuSubqueryBroadcastMeta.scala | 35 +++++++++++ .../spark/rapids/DynamicPruningSuite.scala | 2 +- 4 files changed, 123 insertions(+), 33 deletions(-) create mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala create mode 100644 sql-plugin/src/main/spark400/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExec.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExec.scala index 1861a9f2515..72ed0e79504 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExec.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,14 +41,14 @@ import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.ThreadUtils -class GpuSubqueryBroadcastMeta( +abstract class GpuSubqueryBroadcastMetaBase( s: SubqueryBroadcastExec, conf: RapidsConf, p: Option[RapidsMeta[_, _, _]], r: DataFromReplacementRule) extends SparkPlanMeta[SubqueryBroadcastExec](s, conf, p, r) { - private var broadcastBuilder: () => SparkPlan = _ + protected var broadcastBuilder: () => SparkPlan = _ override val childExprs: Seq[BaseExprMeta[_]] = Nil @@ -140,13 +140,8 @@ class GpuSubqueryBroadcastMeta( */ override def convertToCpu(): SparkPlan = s - override def convertToGpu(): GpuExec = { - GpuSubqueryBroadcastExec(s.name, s.index, s.buildKeys, broadcastBuilder())( - getBroadcastModeKeyExprs) - } - /** Extract the broadcast mode key expressions if there are any. */ - private def getBroadcastModeKeyExprs: Option[Seq[Expression]] = { + protected def getBroadcastModeKeyExprs: Option[Seq[Expression]] = { val broadcastMode = s.child match { case b: BroadcastExchangeExec => b.mode @@ -170,7 +165,7 @@ class GpuSubqueryBroadcastMeta( case class GpuSubqueryBroadcastExec( name: String, - index: Int, + indices: Seq[Int], buildKeys: Seq[Expression], child: SparkPlan)(modeKeys: Option[Seq[Expression]]) extends ShimBaseSubqueryExec with GpuExec with ShimUnaryExecNode { @@ -182,16 +177,18 @@ case class GpuSubqueryBroadcastExec( // correctly report the output length, so that `InSubqueryExec` can know it's the single-column // execution mode, not multi-column. override def output: Seq[Attribute] = { - val key = buildKeys(index) - val name = key match { - case n: NamedExpression => - n.name - case cast: Cast if cast.child.isInstanceOf[NamedExpression] => - cast.child.asInstanceOf[NamedExpression].name - case _ => - "key" + indices.map { index => + val key = buildKeys(index) + val name = key match { + case n: NamedExpression => + n.name + case cast: Cast if cast.child.isInstanceOf[NamedExpression] => + cast.child.asInstanceOf[NamedExpression].name + case _ => + "key" + } + AttributeReference(name, key.dataType, key.nullable)() } - Seq(AttributeReference(name, key.dataType, key.nullable)()) } override lazy val additionalMetrics: Map[String, GpuMetric] = Map( @@ -200,7 +197,7 @@ case class GpuSubqueryBroadcastExec( override def doCanonicalize(): SparkPlan = { val keys = buildKeys.map(k => QueryPlan.normalizeExpressions(k, child.output)) - GpuSubqueryBroadcastExec("dpp", index, keys, child.canonicalized)(modeKeys) + GpuSubqueryBroadcastExec("dpp", indices, keys, child.canonicalized)(modeKeys) } @transient @@ -235,28 +232,30 @@ case class GpuSubqueryBroadcastExec( // are being extracted. The CPU already has the key projections applied in the broadcast // data and thus does not have similar logic here. val broadcastModeProject = modeKeys.map { keyExprs => - val keyExpr = if (GpuHashJoin.canRewriteAsLongType(buildKeys)) { + val exprs = if (GpuHashJoin.canRewriteAsLongType(buildKeys)) { // in this case, there is only 1 key expression since it's a packed version that encompasses // multiple integral values into a single long using bit logic. In CPU Spark, the broadcast // would create a LongHashedRelation instead of a standard HashedRelation. - keyExprs.head + indices.map { _ => keyExprs.head } } else { - keyExprs(index) + indices.map { idx => keyExprs(idx) } } - UnsafeProjection.create(keyExpr) + UnsafeProjection.create(exprs) } - // Use the single output of the broadcast mode projection if it exists - val rowProjectIndex = if (broadcastModeProject.isDefined) 0 else index - val rowExpr = if (GpuHashJoin.canRewriteAsLongType(buildKeys)) { + val rowExprs = if (GpuHashJoin.canRewriteAsLongType(buildKeys)) { // Since this is the expected output for a LongHashedRelation, we can extract the key from the - // long packed key using bit logic, using this method available in HashJoin to give us the - // correct key expression. - HashJoin.extractKeyExprAt(buildKeys, index) + // long packed key using bit logic, using this method available in HashJoin to give us the + // correct key expression. + indices.map { idx => HashJoin.extractKeyExprAt(buildKeys, idx) } } else { - BoundReference(rowProjectIndex, buildKeys(index).dataType, buildKeys(index).nullable) + indices.map { idx => + // Use the single output of the broadcast mode projection if it exists + val rowProjectIndex = if (broadcastModeProject.isDefined) 0 else idx + BoundReference(rowProjectIndex, buildKeys(idx).dataType, buildKeys(idx).nullable) + } } - val rowProject = UnsafeProjection.create(rowExpr) + val rowProject = UnsafeProjection.create(rowExprs) // Deserializes the batch on the host. Then, transforms it to rows and performs row-wise // projection. We should NOT run any device operation on the driver node. diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala new file mode 100644 index 00000000000..9bcfa33ab87 --- /dev/null +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/*** spark-rapids-shim-json-lines +{"spark": "311"} +{"spark": "312"} +{"spark": "313"} +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.execution + +import com.nvidia.spark.rapids.{DataFromReplacementRule, GpuExec, RapidsConf, RapidsMeta} + +import org.apache.spark.sql.execution.SubqueryBroadcastExec + +class GpuSubqueryBroadcastMeta( + s: SubqueryBroadcastExec, + conf: RapidsConf, + p: Option[RapidsMeta[_, _, _]], + r: DataFromReplacementRule) extends + GpuSubqueryBroadcastMetaBase(s, conf, p, r) { + override def convertToGpu(): GpuExec = { + GpuSubqueryBroadcastExec(s.name, Seq(s.index), s.buildKeys, broadcastBuilder())( + getBroadcastModeKeyExprs) + } +} diff --git a/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala b/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala new file mode 100644 index 00000000000..c16564f523e --- /dev/null +++ b/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/*** spark-rapids-shim-json-lines +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.execution + +import com.nvidia.spark.rapids.{DataFromReplacementRule, GpuExec, RapidsConf, RapidsMeta} + +import org.apache.spark.sql.execution.SubqueryBroadcastExec + +class GpuSubqueryBroadcastMeta( + s: SubqueryBroadcastExec, + conf: RapidsConf, + p: Option[RapidsMeta[_, _, _]], + r: DataFromReplacementRule) extends + GpuSubqueryBroadcastMetaBase(s, conf, p, r) { + override def convertToGpu(): GpuExec = { + GpuSubqueryBroadcastExec(s.name, s.indices, s.buildKeys, broadcastBuilder())( + getBroadcastModeKeyExprs) + } +} diff --git a/tests/src/test/spark321/scala/com/nvidia/spark/rapids/DynamicPruningSuite.scala b/tests/src/test/spark321/scala/com/nvidia/spark/rapids/DynamicPruningSuite.scala index 2d4156d1b3b..722e5bb215b 100644 --- a/tests/src/test/spark321/scala/com/nvidia/spark/rapids/DynamicPruningSuite.scala +++ b/tests/src/test/spark321/scala/com/nvidia/spark/rapids/DynamicPruningSuite.scala @@ -66,7 +66,7 @@ class DynamicPruningSuite // NOTE: We remove the AdaptiveSparkPlanExec since we can't re-run the new plan // under AQE because that fundamentally requires some rewrite and stage // ordering which we can't do for this test. - case GpuSubqueryBroadcastExec(name, index, buildKeys, child) => + case GpuSubqueryBroadcastExec(name, Seq(index), buildKeys, child) => val newChild = child match { case a @ AdaptiveSparkPlanExec(_, _, _, _, _) => (new GpuTransitionOverrides()).apply(ColumnarToRowExec(a.executedPlan)) From e23bf381a74237f882d94063b790c3e045e23ed8 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Thu, 30 May 2024 08:18:37 -0700 Subject: [PATCH 06/79] Unarchive Spark test jar for spark.read(ability) (#10946) Closes #10875 Contributes to #10773 Unjar, cache, and share the test jar content among all test suites from the same jar Test: ```bash mvn package -Dbuildver=330 -pl tests -am -Dsuffixes='.*\.RapidsJsonSuite' ``` Signed-off-by: Gera Shegalov --- .../sql/rapids/suites/RapidsJsonSuite.scala | 4 -- .../utils/RapidsSQLTestsBaseTrait.scala | 46 ++++++++++++++++++- .../sql/rapids/utils/RapidsTestSettings.scala | 9 ---- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsJsonSuite.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsJsonSuite.scala index 3e9f685dfdc..ef9ae630dfd 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsJsonSuite.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsJsonSuite.scala @@ -31,10 +31,6 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap class RapidsJsonSuite extends JsonSuite with RapidsSQLTestsBaseTrait with RapidsJsonConfTrait { - /** Returns full path to the given file in the resource folder */ - override protected def testFile(fileName: String): String = { - getWorkspaceFilePath("sql", "core", "src", "test", "resources").toString + "/" + fileName - } } class RapidsJsonV1Suite extends RapidsJsonSuite with RapidsSQLTestsBaseTrait { diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsSQLTestsBaseTrait.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsSQLTestsBaseTrait.scala index 6db9e8b71a6..f8b9d21d169 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsSQLTestsBaseTrait.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsSQLTestsBaseTrait.scala @@ -21,27 +21,36 @@ package org.apache.spark.sql.rapids.utils import java.util.{Locale, TimeZone} +import org.apache.hadoop.fs.FileUtil import org.scalactic.source.Position import org.scalatest.Tag import org.apache.spark.SparkConf +import org.apache.spark.internal.Logging import org.apache.spark.internal.config.Tests.IS_TESTING import org.apache.spark.sql.DataFrame import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, ShuffleQueryStageExec} +import org.apache.spark.sql.rapids.execution.TrampolineUtil import org.apache.spark.sql.rapids.utils.RapidsTestConstants.RAPIDS_TEST import org.apache.spark.sql.test.SharedSparkSession /** Basic trait for Rapids SQL test cases. */ trait RapidsSQLTestsBaseTrait extends SharedSparkSession with RapidsTestsBaseTrait { - protected override def afterAll(): Unit = { // SparkFunSuite will set this to true, and forget to reset to false System.clearProperty(IS_TESTING.key) super.afterAll() } + override protected def testFile(fileName: String): String = { + import RapidsSQLTestsBaseTrait.sparkTestResourcesDir + + java.nio.file.Paths.get(sparkTestResourcesDir(getClass).toString, fileName) + .toString + } + protected def testRapids(testName: String, testTag: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { test(RAPIDS_TEST + testName, testTag: _*)(testFun) @@ -107,7 +116,40 @@ trait RapidsSQLTestsBaseTrait extends SharedSparkSession with RapidsTestsBaseTra } } -object RapidsSQLTestsBaseTrait { +object RapidsSQLTestsBaseTrait extends Logging { + private val resourceMap = scala.collection.mutable.Map.empty[String, java.nio.file.Path] + private val testJarUrlRegex = raw"jar:file:(/.*-tests.jar)!.*".r + TrampolineUtil.addShutdownHook(10000, () => { + resourceMap.valuesIterator.foreach { dirPath => + logWarning(s"Deleting expanded test jar dir $dirPath") + FileUtil.fullyDelete(dirPath.toFile) + } + }) + + private def expandJar(jarPath: String): java.nio.file.Path = { + val jarFile = new java.io.File(jarPath) + val destDir = java.nio.file.Files.createTempDirectory(jarFile.getName + ".expanded") + logWarning(s"Registering $destDir for deletion on exit") + FileUtil.unZip(jarFile, destDir.toFile) + destDir + } + + def sparkTestResourcesDir(testClass: Class[_]): java.nio.file.Path = { + var sparkTestClass = testClass + while (sparkTestClass.getName.contains("rapids")) { + sparkTestClass = sparkTestClass.getSuperclass + } + val sparkTestClassResource = "/" + sparkTestClass.getName.replace(".", "/") + ".class" + val resourceURL = sparkTestClass.getResource(sparkTestClassResource).toString + val resourceJar = resourceURL match { + case testJarUrlRegex(testJarPath) => testJarPath + case _ => sys.error(s"Could not extract tests jar path from $resourceURL") + } + this.synchronized { + resourceMap.getOrElseUpdate(resourceJar, expandJar(resourceJar)) + } + } + def nativeSparkConf(origin: SparkConf, warehouse: String): SparkConf = { // Timezone is fixed to UTC to allow timestamps to work by default TimeZone.setDefault(TimeZone.getTimeZone("UTC")) diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala index 3ccd478d368..8fc86cc6dce 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala @@ -66,15 +66,6 @@ class RapidsTestSettings extends BackendTestSettings { enableSuite[RapidsJsonSuite] .exclude("Casting long as timestamp", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) .exclude("Write timestamps correctly with timestampFormat option and timeZone option", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) - .exclude("SPARK-23723: json in UTF-16 with BOM", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) - .exclude("SPARK-23723: multi-line json in UTF-32BE with BOM", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) - .exclude("SPARK-23723: Use user's encoding in reading of multi-line json in UTF-16LE", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) - .exclude("SPARK-23723: Unsupported encoding name", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) - .exclude("SPARK-23723: checking that the encoding option is case agnostic", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) - .exclude("SPARK-23723: specified encoding is not matched to actual encoding", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) - .exclude("SPARK-23724: lineSep should be set if encoding if different from UTF-8", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) - .exclude("SPARK-31716: inferring should handle malformed input", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) - .exclude("SPARK-24190: restrictions for JSONOptions in read", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) .exclude("exception mode for parsing date/timestamp string", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) enableSuite[RapidsMathFunctionsSuite] enableSuite[RapidsRegexpExpressionsSuite] From a7cdaa9eff17293cae9cee04d16b07d924fd40eb Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Thu, 30 May 2024 08:49:53 -0700 Subject: [PATCH 07/79] Added Shim for BatchScanExec to Support Spark 4.0 [databricks] (#10944) * Added shim for BatchScanExec to support Spark 4.0 Signed-off-by: Raza Jafri * fixed the failing shim --------- Signed-off-by: Raza Jafri --- .../spark/rapids/shims/GpuBatchScanExec.scala | 1 - .../rapids/shims/BatchScanExecMeta.scala | 52 +--- .../rapids/shims/BatchScanExecMetaBase.scala | 81 ++++++ .../rapids/shims/BatchScanExecMeta.scala | 38 +++ .../spark/rapids/shims/GpuBatchScanExec.scala | 269 ++++++++++++++++++ 5 files changed, 389 insertions(+), 52 deletions(-) create mode 100644 sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMetaBase.scala create mode 100644 sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala create mode 100644 sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala index 39f42d8b833..5fb252524fd 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala @@ -22,7 +22,6 @@ {"spark": "343"} {"spark": "350"} {"spark": "351"} -{"spark": "400"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala index 4bbc4644241..4b29de25bf0 100644 --- a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala +++ b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala @@ -17,68 +17,18 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} {"spark": "351"} -{"spark": "400"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims import com.nvidia.spark.rapids._ -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.datasources.v2.BatchScanExec class BatchScanExecMeta(p: BatchScanExec, conf: RapidsConf, parent: Option[RapidsMeta[_, _, _]], rule: DataFromReplacementRule) - extends SparkPlanMeta[BatchScanExec](p, conf, parent, rule) { - // Replaces SubqueryBroadcastExec inside dynamic pruning filters with GPU counterpart - // if possible. Instead regarding filters as childExprs of current Meta, we create - // a new meta for SubqueryBroadcastExec. The reason is that the GPU replacement of - // BatchScanExec is independent from the replacement of the runtime filters. It is - // possible that the BatchScanExec is on the CPU, while the dynamic runtime filters - // are on the GPU. And vice versa. - private lazy val runtimeFilters = { - val convertBroadcast = (bc: SubqueryBroadcastExec) => { - val meta = GpuOverrides.wrapAndTagPlan(bc, conf) - meta.tagForExplain() - meta.convertIfNeeded().asInstanceOf[BaseSubqueryExec] - } - wrapped.runtimeFilters.map { filter => - filter.transformDown { - case dpe @ DynamicPruningExpression(inSub: InSubqueryExec) => - inSub.plan match { - case bc: SubqueryBroadcastExec => - dpe.copy(inSub.copy(plan = convertBroadcast(bc))) - case reuse @ ReusedSubqueryExec(bc: SubqueryBroadcastExec) => - dpe.copy(inSub.copy(plan = reuse.copy(convertBroadcast(bc)))) - case _ => - dpe - } - } - } - } - - override val childExprs: Seq[BaseExprMeta[_]] = { - // We want to leave the runtime filters as CPU expressions - p.output.map(GpuOverrides.wrapExpr(_, conf, Some(this))) - } - - override val childScans: scala.Seq[ScanMeta[_]] = - Seq(GpuOverrides.wrapScan(p.scan, conf, Some(this))) - - override def tagPlanForGpu(): Unit = { - if (!p.runtimeFilters.isEmpty && !childScans.head.supportsRuntimeFilters) { - willNotWorkOnGpu("runtime filtering (DPP) is not supported for this scan") - } - } - - override def convertToCpu(): SparkPlan = { - val cpu = wrapped.copy(runtimeFilters = runtimeFilters) - cpu.copyTagsFrom(wrapped) - cpu - } - + extends BatchScanExecMetaBase(p, conf, parent, rule) { override def convertToGpu(): GpuExec = { val spj = p.spjParams GpuBatchScanExec(p.output, childScans.head.convertToGpu(), runtimeFilters, diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMetaBase.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMetaBase.scala new file mode 100644 index 00000000000..914702a289c --- /dev/null +++ b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMetaBase.scala @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "350"} +{"spark": "351"} +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import com.nvidia.spark.rapids._ + +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.execution._ +import org.apache.spark.sql.execution.datasources.v2.BatchScanExec + +abstract class BatchScanExecMetaBase(p: BatchScanExec, + conf: RapidsConf, + parent: Option[RapidsMeta[_, _, _]], + rule: DataFromReplacementRule) + extends SparkPlanMeta[BatchScanExec](p, conf, parent, rule) { + // Replaces SubqueryBroadcastExec inside dynamic pruning filters with GPU counterpart + // if possible. Instead regarding filters as childExprs of current Meta, we create + // a new meta for SubqueryBroadcastExec. The reason is that the GPU replacement of + // BatchScanExec is independent from the replacement of the runtime filters. It is + // possible that the BatchScanExec is on the CPU, while the dynamic runtime filters + // are on the GPU. And vice versa. + protected lazy val runtimeFilters = { + val convertBroadcast = (bc: SubqueryBroadcastExec) => { + val meta = GpuOverrides.wrapAndTagPlan(bc, conf) + meta.tagForExplain() + meta.convertIfNeeded().asInstanceOf[BaseSubqueryExec] + } + wrapped.runtimeFilters.map { filter => + filter.transformDown { + case dpe @ DynamicPruningExpression(inSub: InSubqueryExec) => + inSub.plan match { + case bc: SubqueryBroadcastExec => + dpe.copy(inSub.copy(plan = convertBroadcast(bc))) + case reuse @ ReusedSubqueryExec(bc: SubqueryBroadcastExec) => + dpe.copy(inSub.copy(plan = reuse.copy(convertBroadcast(bc)))) + case _ => + dpe + } + } + } + } + + override val childExprs: Seq[BaseExprMeta[_]] = { + // We want to leave the runtime filters as CPU expressions + p.output.map(GpuOverrides.wrapExpr(_, conf, Some(this))) + } + + override val childScans: scala.Seq[ScanMeta[_]] = + Seq(GpuOverrides.wrapScan(p.scan, conf, Some(this))) + + override def tagPlanForGpu(): Unit = { + if (!p.runtimeFilters.isEmpty && !childScans.head.supportsRuntimeFilters) { + willNotWorkOnGpu("runtime filtering (DPP) is not supported for this scan") + } + } + + override def convertToCpu(): SparkPlan = { + val cpu = wrapped.copy(runtimeFilters = runtimeFilters) + cpu.copyTagsFrom(wrapped) + cpu + } +} diff --git a/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala new file mode 100644 index 00000000000..e6c26eb65b8 --- /dev/null +++ b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import com.nvidia.spark.rapids._ + +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.execution._ +import org.apache.spark.sql.execution.datasources.v2.BatchScanExec + +class BatchScanExecMeta(p: BatchScanExec, + conf: RapidsConf, + parent: Option[RapidsMeta[_, _, _]], + rule: DataFromReplacementRule) + extends BatchScanExecMetaBase(p, conf, parent, rule) { + override def convertToGpu(): GpuExec = { + val spj = p.spjParams + GpuBatchScanExec(p.output, childScans.head.convertToGpu(), runtimeFilters, + p.ordering, p.table, spj) + } +} diff --git a/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala new file mode 100644 index 00000000000..3c2b649339b --- /dev/null +++ b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import com.google.common.base.Objects +import com.nvidia.spark.rapids.GpuScan + +import org.apache.spark.SparkException +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, DynamicPruningExpression, Expression, Literal, RowOrdering, SortOrder} +import org.apache.spark.sql.catalyst.plans.QueryPlan +import org.apache.spark.sql.catalyst.plans.physical.{KeyGroupedPartitioning, KeyGroupedShuffleSpec, Partitioning, SinglePartition} +import org.apache.spark.sql.catalyst.util.{truncatedString, InternalRowComparableWrapper} +import org.apache.spark.sql.connector.catalog.Table +import org.apache.spark.sql.connector.read._ +import org.apache.spark.sql.execution.datasources.rapids.DataSourceStrategyUtils +import org.apache.spark.sql.execution.datasources.v2.{DataSourceRDD, StoragePartitionJoinParams} +import org.apache.spark.sql.internal.SQLConf + +case class GpuBatchScanExec( + output: Seq[AttributeReference], + @transient scan: GpuScan, + runtimeFilters: Seq[Expression] = Seq.empty, + ordering: Option[Seq[SortOrder]] = None, + @transient table: Table, + spjParams: StoragePartitionJoinParams = StoragePartitionJoinParams() + ) extends GpuBatchScanExecBase(scan, runtimeFilters) { + + @transient lazy val batch: Batch = if (scan == null) null else scan.toBatch + // TODO: unify the equal/hashCode implementation for all data source v2 query plans. + override def equals(other: Any): Boolean = other match { + case other: GpuBatchScanExec => + this.batch != null && this.batch == other.batch && + this.runtimeFilters == other.runtimeFilters && + this.spjParams == other.spjParams + case _ => + false + } + + override def hashCode(): Int = Objects.hashCode(batch, runtimeFilters) + + @transient override lazy val inputPartitions: Seq[InputPartition] = + batch.planInputPartitions() + + @transient override protected lazy val filteredPartitions: Seq[Seq[InputPartition]] = { + val dataSourceFilters = runtimeFilters.flatMap { + case DynamicPruningExpression(e) => DataSourceStrategyUtils.translateRuntimeFilter(e) + case _ => None + } + + if (dataSourceFilters.nonEmpty) { + val originalPartitioning = outputPartitioning + + // the cast is safe as runtime filters are only assigned if the scan can be filtered + val filterableScan = scan.asInstanceOf[SupportsRuntimeV2Filtering] + filterableScan.filter(dataSourceFilters.toArray) + + // call toBatch again to get filtered partitions + val newPartitions = scan.toBatch.planInputPartitions() + + originalPartitioning match { + case p: KeyGroupedPartitioning => + if (newPartitions.exists(!_.isInstanceOf[HasPartitionKey])) { + throw new SparkException("Data source must have preserved the original partitioning " + + "during runtime filtering: not all partitions implement HasPartitionKey after " + + "filtering") + } + + val newPartitionValues = newPartitions.map(partition => + InternalRowComparableWrapper(partition.asInstanceOf[HasPartitionKey], p.expressions)) + .toSet + val oldPartitionValues = p.partitionValues + .map(partition => InternalRowComparableWrapper(partition, p.expressions)).toSet + // We require the new number of partition values to be equal or less than the old number + // of partition values here. In the case of less than, empty partitions will be added for + // those missing values that are not present in the new input partitions. + if (oldPartitionValues.size < newPartitionValues.size) { + throw new SparkException("During runtime filtering, data source must either report " + + "the same number of partition values, or a subset of partition values from the " + + s"original. Before: ${oldPartitionValues.size} partition values. " + + s"After: ${newPartitionValues.size} partition values") + } + + if (!newPartitionValues.forall(oldPartitionValues.contains)) { + throw new SparkException("During runtime filtering, data source must not report new " + + "partition values that are not present in the original partitioning.") + } + groupPartitions(newPartitions) + .map(_.groupedParts.map(_.parts)).getOrElse(Seq.empty) + + case _ => + // no validation is needed as the data source did not report any specific partitioning + newPartitions.map(Seq(_)) + } + + } else { + partitions + } + } + + override def outputPartitioning: Partitioning = { + super.outputPartitioning match { + case k: KeyGroupedPartitioning if spjParams.commonPartitionValues.isDefined => + // We allow duplicated partition values if + // `spark.sql.sources.v2.bucketing.partiallyClusteredDistribution.enabled` is true + val newPartValues = spjParams.commonPartitionValues.get.flatMap { + case (partValue, numSplits) => Seq.fill(numSplits)(partValue) + } + val expressions = spjParams.joinKeyPositions match { + case Some(projectionPositions) => projectionPositions.map(i => k.expressions(i)) + case _ => k.expressions + } + k.copy(expressions = expressions, numPartitions = newPartValues.length, + partitionValues = newPartValues) + case p => p + } + } + + override lazy val readerFactory: PartitionReaderFactory = batch.createReaderFactory() + + override lazy val inputRDD: RDD[InternalRow] = { + val rdd = if (filteredPartitions.isEmpty && outputPartitioning == SinglePartition) { + // return an empty RDD with 1 partition if dynamic filtering removed the only split + sparkContext.parallelize(Array.empty[InternalRow], 1) + } else { + val finalPartitions = outputPartitioning match { + case p: KeyGroupedPartitioning => + assert(spjParams.keyGroupedPartitioning.isDefined) + val expressions = spjParams.keyGroupedPartitioning.get + + // Re-group the input partitions if we are projecting on a subset of join keys + val (groupedPartitions, partExpressions) = spjParams.joinKeyPositions match { + case Some(projectPositions) => + val projectedExpressions = projectPositions.map(i => expressions(i)) + val parts = filteredPartitions.flatten.groupBy(part => { + val row = part.asInstanceOf[HasPartitionKey].partitionKey() + val projectedRow = KeyGroupedPartitioning.project( + expressions, projectPositions, row) + InternalRowComparableWrapper(projectedRow, projectedExpressions) + }).map { case (wrapper, splits) => (wrapper.row, splits) }.toSeq + (parts, projectedExpressions) + case _ => + val groupedParts = filteredPartitions.map(splits => { + assert(splits.nonEmpty && splits.head.isInstanceOf[HasPartitionKey]) + (splits.head.asInstanceOf[HasPartitionKey].partitionKey(), splits) + }) + (groupedParts, expressions) + } + + // Also re-group the partitions if we are reducing compatible partition expressions + val finalGroupedPartitions = spjParams.reducers match { + case Some(reducers) => + val result = groupedPartitions.groupBy { case (row, _) => + KeyGroupedShuffleSpec.reducePartitionValue(row, partExpressions, reducers) + }.map { case (wrapper, splits) => (wrapper.row, splits.flatMap(_._2)) }.toSeq + val rowOrdering = RowOrdering.createNaturalAscendingOrdering( + partExpressions.map(_.dataType)) + result.sorted(rowOrdering.on((t: (InternalRow, _)) => t._1)) + case _ => groupedPartitions + } + + // When partially clustered, the input partitions are not grouped by partition + // values. Here we'll need to check `commonPartitionValues` and decide how to group + // and replicate splits within a partition. + if (spjParams.commonPartitionValues.isDefined && spjParams.applyPartialClustering) { + // A mapping from the common partition values to how many splits the partition + // should contain. + val commonPartValuesMap = spjParams.commonPartitionValues + .get + .map(t => (InternalRowComparableWrapper(t._1, partExpressions), t._2)) + .toMap + val nestGroupedPartitions = finalGroupedPartitions.map { case (partValue, splits) => + // `commonPartValuesMap` should contain the part value since it's the super set. + val numSplits = commonPartValuesMap + .get(InternalRowComparableWrapper(partValue, partExpressions)) + assert(numSplits.isDefined, s"Partition value $partValue does not exist in " + + "common partition values from Spark plan") + + val newSplits = if (spjParams.replicatePartitions) { + // We need to also replicate partitions according to the other side of join + Seq.fill(numSplits.get)(splits) + } else { + // Not grouping by partition values: this could be the side with partially + // clustered distribution. Because of dynamic filtering, we'll need to check if + // the final number of splits of a partition is smaller than the original + // number, and fill with empty splits if so. This is necessary so that both + // sides of a join will have the same number of partitions & splits. + splits.map(Seq(_)).padTo(numSplits.get, Seq.empty) + } + (InternalRowComparableWrapper(partValue, partExpressions), newSplits) + } + + // Now fill missing partition keys with empty partitions + val partitionMapping = nestGroupedPartitions.toMap + spjParams.commonPartitionValues.get.flatMap { + case (partValue, numSplits) => + // Use empty partition for those partition values that are not present. + partitionMapping.getOrElse( + InternalRowComparableWrapper(partValue, partExpressions), + Seq.fill(numSplits)(Seq.empty)) + } + } else { + // either `commonPartitionValues` is not defined, or it is defined but + // `applyPartialClustering` is false. + val partitionMapping = finalGroupedPartitions.map { case (partValue, splits) => + InternalRowComparableWrapper(partValue, partExpressions) -> splits + }.toMap + + // In case `commonPartitionValues` is not defined (e.g., SPJ is not used), there + // could exist duplicated partition values, as partition grouping is not done + // at the beginning and postponed to this method. It is important to use unique + // partition values here so that grouped partitions won't get duplicated. + p.uniquePartitionValues.map { partValue => + // Use empty partition for those partition values that are not present + partitionMapping.getOrElse( + InternalRowComparableWrapper(partValue, partExpressions), Seq.empty) + } + } + + case _ => filteredPartitions + } + + new DataSourceRDD( + sparkContext, finalPartitions, readerFactory, supportsColumnar, customMetrics) + } + postDriverMetrics() + rdd + } + + override def keyGroupedPartitioning: Option[Seq[Expression]] = + spjParams.keyGroupedPartitioning + + override def doCanonicalize(): GpuBatchScanExec = { + this.copy( + output = output.map(QueryPlan.normalizeExpressions(_, output)), + runtimeFilters = QueryPlan.normalizePredicates( + runtimeFilters.filterNot(_ == DynamicPruningExpression(Literal.TrueLiteral)), + output)) + } + + override def simpleString(maxFields: Int): String = { + val truncatedOutputString = truncatedString(output, "[", ", ", "]", maxFields) + val runtimeFiltersString = s"RuntimeFilters: ${runtimeFilters.mkString("[", ",", "]")}" + val result = s"$nodeName$truncatedOutputString ${scan.description()} $runtimeFiltersString" + redact(result) + } + + override def nodeName: String = { + s"GpuBatchScan ${table.name()}".trim + } +} From 499a45bd2bea4b256eb9b0dbae4720cbedd8a295 Mon Sep 17 00:00:00 2001 From: MithunR Date: Thu, 30 May 2024 14:13:52 -0700 Subject: [PATCH 08/79] [Spark 4.0] Account for `CommandUtils.uncacheTableOrView` signature change. (#10863) * Account for `CommandUtils.uncacheTableOrView` signature change. Fixes #10710. This commit accounts for the changes in the signature of `CommandUtils.uncacheTableOrView` in Apache Spark 4.0. (See [SPARK-47191](https://github.com/apache/spark/pull/45289).) Signed-off-by: MithunR * Removed unnecessary base class. --------- Signed-off-by: MithunR --- .../hive/rapids/shims/CommandUtilsShim.scala | 57 +++++++++++++++++++ .../rapids/shims/GpuInsertIntoHiveTable.scala | 2 +- .../rapids/shims/GpuInsertIntoHiveTable.scala | 2 +- .../hive/rapids/shims/CommandUtilsShim.scala | 33 +++++++++++ 4 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala create mode 100644 sql-plugin/src/main/spark400/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala new file mode 100644 index 00000000000..1e1ac57aa60 --- /dev/null +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "311"} +{"spark": "312"} +{"spark": "313"} +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.hive.rapids.shims + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.execution.command.CommandUtils + +object CommandUtilsShim { + + // Shim for CommandUtils.uncacheTableOrView, whose signature changed in Apache Spark 4.0. + def uncacheTableOrView(sparkSession: SparkSession, tableId: TableIdentifier): Unit = { + CommandUtils.uncacheTableOrView(sparkSession, tableId.quotedString) + } + +} \ No newline at end of file diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala index 92fb72801c8..2af89bf1170 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala @@ -137,7 +137,7 @@ case class GpuInsertIntoHiveTable( } // un-cache this table. - CommandUtils.uncacheTableOrView(sparkSession, table.identifier.quotedString) + CommandUtilsShim.uncacheTableOrView(sparkSession, table.identifier) sparkSession.sessionState.catalog.refreshTable(table.identifier) CommandUtils.updateTableStats(sparkSession, table) diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala index 9105ab50e1e..28b8033389a 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala @@ -127,7 +127,7 @@ case class GpuInsertIntoHiveTable( } // un-cache this table. - CommandUtils.uncacheTableOrView(sparkSession, table.identifier.quotedString) + CommandUtilsShim.uncacheTableOrView(sparkSession, table.identifier) sparkSession.sessionState.catalog.refreshTable(table.identifier) CommandUtils.updateTableStats(sparkSession, table) diff --git a/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala b/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala new file mode 100644 index 00000000000..f5858e4cfd6 --- /dev/null +++ b/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.hive.rapids.shims + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.execution.command.CommandUtils + +object CommandUtilsShim { + + // Shim for CommandUtils.uncacheTableOrView, whose signature changed in Apache Spark 4.0. + def uncacheTableOrView(sparkSession: SparkSession, tableId: TableIdentifier): Unit = { + CommandUtils.uncacheTableOrView(sparkSession, tableId) + } + +} \ No newline at end of file From 4024ef66d269864d5d9af5e9d7c9015f6982629c Mon Sep 17 00:00:00 2001 From: Liangcai Li Date: Fri, 31 May 2024 09:47:12 +0800 Subject: [PATCH 09/79] GpuInsertIntoHiveTable supports parquet format (#10912) This is a new feature adding the parquet support for GpuInsertIntoHiveTable, who only supports text write now. And this feature is tested by the new added tests in this PR. --------- Signed-off-by: Firestarman Co-authored-by: Jason Lowe --- .../main/python/hive_parquet_write_test.py | 176 ++++++++++++++++++ .../spark/rapids/GpuParquetFileFormat.scala | 4 +- ...leFormat.scala => GpuHiveFileFormat.scala} | 149 ++++++++++++--- .../rapids/shims/GpuInsertIntoHiveTable.scala | 17 +- .../rapids/shims/GpuInsertIntoHiveTable.scala | 17 +- 5 files changed, 322 insertions(+), 41 deletions(-) create mode 100644 integration_tests/src/main/python/hive_parquet_write_test.py rename sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/{GpuHiveTextFileFormat.scala => GpuHiveFileFormat.scala} (54%) diff --git a/integration_tests/src/main/python/hive_parquet_write_test.py b/integration_tests/src/main/python/hive_parquet_write_test.py new file mode 100644 index 00000000000..e74a99f43c7 --- /dev/null +++ b/integration_tests/src/main/python/hive_parquet_write_test.py @@ -0,0 +1,176 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from asserts import assert_gpu_and_cpu_sql_writes_are_equal_collect +from conftest import is_databricks_runtime +from data_gen import * +from hive_write_test import _restricted_timestamp +from marks import allow_non_gpu, ignore_order +from spark_session import with_cpu_session, is_before_spark_320 + +# Disable the meta conversion from Hive write to FrameData write in Spark, to test +# "GpuInsertIntoHiveTable" for Parquet write. +_write_to_hive_conf = {"spark.sql.hive.convertMetastoreParquet": False} + +_hive_basic_gens = [ + byte_gen, short_gen, int_gen, long_gen, float_gen, double_gen, string_gen, boolean_gen, + DateGen(start=date(1590, 1, 1)), _restricted_timestamp(), + DecimalGen(precision=19, scale=1, nullable=True), + DecimalGen(precision=23, scale=5, nullable=True), + DecimalGen(precision=36, scale=3, nullable=True)] + +_hive_basic_struct_gen = StructGen( + [['c'+str(ind), c_gen] for ind, c_gen in enumerate(_hive_basic_gens)]) + +_hive_struct_gens = [ + _hive_basic_struct_gen, + StructGen([['child0', byte_gen], ['child1', _hive_basic_struct_gen]]), + StructGen([['child0', ArrayGen(short_gen)], ['child1', double_gen]])] + +_hive_array_gens = [ArrayGen(sub_gen) for sub_gen in _hive_basic_gens] + [ + ArrayGen(ArrayGen(short_gen, max_length=10), max_length=10), + ArrayGen(ArrayGen(string_gen, max_length=10), max_length=10), + ArrayGen(StructGen([['child0', byte_gen], ['child1', string_gen], ['child2', float_gen]]))] + +_hive_map_gens = [simple_string_to_string_map_gen] + [MapGen(f(nullable=False), f()) for f in [ + BooleanGen, ByteGen, ShortGen, IntegerGen, LongGen, FloatGen, DoubleGen, + lambda nullable=True: _restricted_timestamp(nullable=nullable), + lambda nullable=True: DateGen(start=date(1590, 1, 1), nullable=nullable), + lambda nullable=True: DecimalGen(precision=19, scale=1, nullable=nullable), + lambda nullable=True: DecimalGen(precision=36, scale=5, nullable=nullable)]] + +_hive_write_gens = [_hive_basic_gens, _hive_struct_gens, _hive_array_gens, _hive_map_gens] + +# ProjectExec falls back on databricks due to no GPU version of "MapFromArrays". +fallback_nodes = ['ProjectExec'] if is_databricks_runtime() else [] + + +@allow_non_gpu(*(non_utc_allow + fallback_nodes)) +@ignore_order(local=True) +@pytest.mark.parametrize("is_ctas", [True, False], ids=['CTAS', 'CTTW']) +@pytest.mark.parametrize("gens", _hive_write_gens, ids=idfn) +def test_write_parquet_into_hive_table(spark_tmp_table_factory, is_ctas, gens): + + def gen_table(spark): + gen_list = [('_c' + str(i), gen) for i, gen in enumerate(gens)] + types_sql_str = ','.join('{} {}'.format( + name, gen.data_type.simpleString()) for name, gen in gen_list) + data_table = spark_tmp_table_factory.get() + gen_df(spark, gen_list).createOrReplaceTempView(data_table) + return data_table, types_sql_str + + (input_table, input_schema) = with_cpu_session(gen_table) + + def write_to_hive_sql(spark, output_table): + if is_ctas: + # Create Table As Select + return [ + "CREATE TABLE {} STORED AS PARQUET AS SELECT * FROM {}".format( + output_table, input_table) + ] + else: + # Create Table Then Write + return [ + "CREATE TABLE {} ({}) STORED AS PARQUET".format(output_table, input_schema), + "INSERT OVERWRITE TABLE {} SELECT * FROM {}".format(output_table, input_table) + ] + + assert_gpu_and_cpu_sql_writes_are_equal_collect( + spark_tmp_table_factory, + write_to_hive_sql, + _write_to_hive_conf) + + +@allow_non_gpu(*non_utc_allow) +@ignore_order(local=True) +@pytest.mark.parametrize("is_static", [True, False], ids=['Static_Partition', 'Dynamic_Partition']) +def test_write_parquet_into_partitioned_hive_table(spark_tmp_table_factory, is_static): + # Generate hive table in Parquet format + def gen_table(spark): + # gen_list = [('_c' + str(i), gen) for i, gen in enumerate(gens)] + dates = [date(2024, 2, 28), date(2024, 2, 27), date(2024, 2, 26)] + gen_list = [('a', int_gen), + ('b', long_gen), + ('c', short_gen), + ('d', string_gen), + ('part', SetValuesGen(DateType(), dates))] + data_table = spark_tmp_table_factory.get() + gen_df(spark, gen_list).createOrReplaceTempView(data_table) + return data_table + + input_table = with_cpu_session(gen_table) + + def partitioned_write_to_hive_sql(spark, output_table): + sql_create_part_table = ( + "CREATE TABLE {} (a INT, b LONG, c SHORT, d STRING) " + "PARTITIONED BY (part DATE) STORED AS PARQUET" + ).format(output_table) + if is_static: + return [ + # sql_1: Create partitioned hive table + sql_create_part_table, + # sql_2: Static partition write only to partition 'par2' + "INSERT OVERWRITE TABLE {} PARTITION (part='2024-02-25') " + "SELECT a, b, c, d FROM {}".format(output_table, input_table) + ] + else: + return [ + # sql_1: Create partitioned hive table + sql_create_part_table, + # sql_2: Dynamic partition write + "INSERT OVERWRITE TABLE {} SELECT * FROM {}".format(output_table, input_table) + ] + all_confs = copy_and_update(_write_to_hive_conf, { + "hive.exec.dynamic.partition.mode": "nonstrict"}) + assert_gpu_and_cpu_sql_writes_are_equal_collect( + spark_tmp_table_factory, + partitioned_write_to_hive_sql, + all_confs) + + +zstd_param = pytest.param('ZSTD', + marks=pytest.mark.skipif(is_before_spark_320(), reason="zstd is not supported before 320")) + +@allow_non_gpu(*(non_utc_allow + fallback_nodes)) +@ignore_order(local=True) +@pytest.mark.parametrize("comp_type", ['UNCOMPRESSED', 'SNAPPY', zstd_param]) +def test_write_compressed_parquet_into_hive_table(spark_tmp_table_factory, comp_type): + # Generate hive table in Parquet format + def gen_table(spark): + gens = _hive_basic_gens + _hive_struct_gens + _hive_array_gens + _hive_map_gens + gen_list = [('_c' + str(i), gen) for i, gen in enumerate(gens)] + types_sql_str = ','.join('{} {}'.format( + name, gen.data_type.simpleString()) for name, gen in gen_list) + data_table = spark_tmp_table_factory.get() + gen_df(spark, gen_list).createOrReplaceTempView(data_table) + return data_table, types_sql_str + + input_table, schema_str = with_cpu_session(gen_table) + + def write_to_hive_sql(spark, output_table): + return [ + # Create table with compression type + "CREATE TABLE {} ({}) STORED AS PARQUET " + "TBLPROPERTIES ('parquet.compression'='{}')".format( + output_table, schema_str, comp_type), + # Insert into table + "INSERT OVERWRITE TABLE {} SELECT * FROM {}".format(output_table, input_table) + ] + + assert_gpu_and_cpu_sql_writes_are_equal_collect( + spark_tmp_table_factory, + write_to_hive_sql, + _write_to_hive_conf) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuParquetFileFormat.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuParquetFileFormat.scala index e8ae977b1f6..25105386b3d 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuParquetFileFormat.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuParquetFileFormat.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -321,7 +321,7 @@ class GpuParquetWriter( new GpuColumnVector(cv.dataType, deepTransformColumn(cv.getBase, cv.dataType)) .asInstanceOf[org.apache.spark.sql.vectorized.ColumnVector] } - new ColumnarBatch(transformedCols) + new ColumnarBatch(transformedCols, batch.numRows()) } } diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveTextFileFormat.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveFileFormat.scala similarity index 54% rename from sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveTextFileFormat.scala rename to sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveFileFormat.scala index 4595ea87ed3..21437a64481 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveTextFileFormat.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveFileFormat.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,8 +17,9 @@ package org.apache.spark.sql.hive.rapids import java.nio.charset.Charset +import java.util.Locale -import ai.rapids.cudf.{CSVWriterOptions, DType, QuoteStyle, Scalar, Table, TableWriter => CudfTableWriter} +import ai.rapids.cudf.{CompressionType, CSVWriterOptions, DType, ParquetWriterOptions, QuoteStyle, Scalar, Table, TableWriter => CudfTableWriter} import com.google.common.base.Charsets import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.Arm.withResource @@ -27,14 +28,85 @@ import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext} import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.hive.rapids.GpuHiveTextFileUtils._ +import org.apache.spark.sql.execution.datasources.parquet.ParquetOptions import org.apache.spark.sql.hive.rapids.shims.GpuInsertIntoHiveTableMeta -import org.apache.spark.sql.types.{DataType, StringType, StructType} +import org.apache.spark.sql.rapids.execution.TrampolineUtil +import org.apache.spark.sql.types.{DataType, Decimal, DecimalType, StringType, StructType} import org.apache.spark.sql.vectorized.ColumnarBatch -object GpuHiveTextFileFormat extends Logging { +object GpuHiveFileFormat extends Logging { + private val parquetOutputFormatClass = + "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat" + private val parquetSerdeClass = + "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe" - private def checkIfEnabled(meta: GpuInsertIntoHiveTableMeta): Unit = { + def tagGpuSupport(meta: GpuInsertIntoHiveTableMeta): Option[ColumnarFileFormat] = { + val insertCmd = meta.wrapped + // Bucketing write + if (insertCmd.table.bucketSpec.isDefined) { + meta.willNotWorkOnGpu("bucketed tables are not supported yet") + } + + // Infer the file format from the serde string, similar as what Spark does in + // RelationConversions for Hive. + val serde = insertCmd.table.storage.serde.getOrElse("").toLowerCase(Locale.ROOT) + val tempFileFormat = if (serde.contains("parquet")) { + // Parquet specific tagging + tagGpuSupportForParquet(meta) + } else { + // Default to text file format + tagGpuSupportForText(meta) + } + + if (meta.canThisBeReplaced) { + Some(tempFileFormat) + } else { + None + } + } + + private def tagGpuSupportForParquet(meta: GpuInsertIntoHiveTableMeta): ColumnarFileFormat = { + val insertCmd = meta.wrapped + val storage = insertCmd.table.storage + + if (storage.outputFormat.getOrElse("") != parquetOutputFormatClass) { + meta.willNotWorkOnGpu(s"unsupported output format found: ${storage.outputFormat}, " + + s"only $parquetOutputFormatClass is currently supported for Parquet") + } + if (storage.serde.getOrElse("") != parquetSerdeClass) { + meta.willNotWorkOnGpu(s"unsupported serde found: ${storage.serde}, " + + s"only $parquetSerdeClass is currently supported for Parquet") + } + + // Decimal type check + val hasIntOrLongBackedDec = insertCmd.query.schema.exists { field => + TrampolineUtil.dataTypeExistsRecursively(field.dataType, { + case dec: DecimalType if dec.precision <= Decimal.MAX_LONG_DIGITS => true + case _ => false + }) + } + if (hasIntOrLongBackedDec) { + meta.willNotWorkOnGpu("decimals that fit in a long are not supported " + + s"for Parquet. Hive always writes decimals as binary arrays but the GPU writes them " + + s"as integral types") + } + + FileFormatChecks.tag(meta, insertCmd.table.schema, ParquetFormatType, WriteFileOp) + + // Compression type + val parquetOptions = new ParquetOptions(insertCmd.table.properties, insertCmd.conf) + val compressionType = + GpuParquetFileFormat.parseCompressionType(parquetOptions.compressionCodecClassName) + .getOrElse { + meta.willNotWorkOnGpu("compression codec " + + s"${parquetOptions.compressionCodecClassName} is not supported for Parquet") + CompressionType.NONE + } + new GpuHiveParquetFileFormat(compressionType) + } + + private def tagGpuSupportForText(meta: GpuInsertIntoHiveTableMeta): ColumnarFileFormat = { + import org.apache.spark.sql.hive.rapids.GpuHiveTextFileUtils._ if (!meta.conf.isHiveDelimitedTextEnabled) { meta.willNotWorkOnGpu("Hive text I/O has been disabled. To enable this, " + s"set ${RapidsConf.ENABLE_HIVE_TEXT} to true") @@ -43,21 +115,16 @@ object GpuHiveTextFileFormat extends Logging { meta.willNotWorkOnGpu("writing Hive delimited text tables has been disabled, " + s"to enable this, set ${RapidsConf.ENABLE_HIVE_TEXT_WRITE} to true") } - } - - def tagGpuSupport(meta: GpuInsertIntoHiveTableMeta) - : Option[ColumnarFileFormat] = { - checkIfEnabled(meta) val insertCommand = meta.wrapped val storage = insertCommand.table.storage if (storage.outputFormat.getOrElse("") != textOutputFormat) { meta.willNotWorkOnGpu(s"unsupported output-format found: ${storage.outputFormat}, " + - s"only $textOutputFormat is currently supported") + s"only $textOutputFormat is currently supported for text") } if (storage.serde.getOrElse("") != lazySimpleSerDe) { meta.willNotWorkOnGpu(s"unsupported serde found: ${storage.serde}, " + - s"only $lazySimpleSerDe is currently supported") + s"only $lazySimpleSerDe is currently supported for text") } val serializationFormat = storage.properties.getOrElse(serializationKey, "1") @@ -86,28 +153,60 @@ object GpuHiveTextFileFormat extends Logging { meta.willNotWorkOnGpu("only UTF-8 is supported as the charset") } - if (insertCommand.table.bucketSpec.isDefined) { - meta.willNotWorkOnGpu("bucketed tables are not supported") - } - - if (insertCommand.conf.getConfString("hive.exec.compress.output", "false").toLowerCase - != "false") { + if (insertCommand.conf.getConfString("hive.exec.compress.output", "false").toBoolean) { meta.willNotWorkOnGpu("compressed output is not supported, " + "set hive.exec.compress.output to false to enable writing Hive text via GPU") } - FileFormatChecks.tag(meta, - insertCommand.table.schema, - HiveDelimitedTextFormatType, - WriteFileOp) + FileFormatChecks.tag(meta, insertCommand.table.schema, HiveDelimitedTextFormatType, + WriteFileOp) - Some(new GpuHiveTextFileFormat()) + new GpuHiveTextFileFormat() } } +class GpuHiveParquetFileFormat(compType: CompressionType) extends ColumnarFileFormat { + + override def prepareWrite(sparkSession: SparkSession, job: Job, + options: Map[String, String], dataSchema: StructType): ColumnarOutputWriterFactory = { + + // Avoid referencing the outer object. + val compressionType = compType + new ColumnarOutputWriterFactory { + override def getFileExtension(context: TaskAttemptContext): String = + compressionType match { + case CompressionType.NONE => ".parquet" + case ct => s".${ct.name().toLowerCase(Locale.ROOT)}.parquet" + } + + override def newInstance(path: String, + dataSchema: StructType, + context: TaskAttemptContext): ColumnarOutputWriter = { + new GpuHiveParquetWriter(path, dataSchema, context, compressionType) + } + } + } +} + +class GpuHiveParquetWriter(override val path: String, dataSchema: StructType, + context: TaskAttemptContext, compType: CompressionType) + extends ColumnarOutputWriter(context, dataSchema, "HiveParquet", true) { + + override protected val tableWriter: CudfTableWriter = { + val optionsBuilder = SchemaUtils + .writerOptionsFromSchema(ParquetWriterOptions.builder(), dataSchema, + writeInt96 = true, // Hive 1.2 write timestamp as INT96 + parquetFieldIdEnabled = false) + .withCompressionType(compType) + Table.writeParquetChunked(optionsBuilder.build(), this) + } + +} + class GpuHiveTextFileFormat extends ColumnarFileFormat with Logging { - override def supportDataType(dataType: DataType): Boolean = isSupportedType(dataType) + override def supportDataType(dataType: DataType): Boolean = + GpuHiveTextFileUtils.isSupportedType(dataType) override def prepareWrite(sparkSession: SparkSession, job: Job, diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala index 2af89bf1170..6d4ca5da7c3 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala @@ -57,7 +57,7 @@ import org.apache.spark.sql.hive.HiveShim.{ShimFileSinkDesc => FileSinkDesc} import org.apache.spark.sql.hive.client.HiveClientImpl import org.apache.spark.sql.hive.client.hive._ import org.apache.spark.sql.hive.execution.InsertIntoHiveTable -import org.apache.spark.sql.hive.rapids.{GpuHiveTextFileFormat, GpuSaveAsHiveFile, RapidsHiveErrors} +import org.apache.spark.sql.hive.rapids.{GpuHiveFileFormat, GpuSaveAsHiveFile, RapidsHiveErrors} import org.apache.spark.sql.vectorized.ColumnarBatch final class GpuInsertIntoHiveTableMeta(cmd: InsertIntoHiveTable, @@ -69,16 +69,17 @@ final class GpuInsertIntoHiveTableMeta(cmd: InsertIntoHiveTable, private var fileFormat: Option[ColumnarFileFormat] = None override def tagSelfForGpuInternal(): Unit = { - // Only Hive delimited text writes are currently supported. - // Check whether that is the format currently in play. - fileFormat = GpuHiveTextFileFormat.tagGpuSupport(this) + fileFormat = GpuHiveFileFormat.tagGpuSupport(this) } override def convertToGpu(): GpuDataWritingCommand = { + val format = fileFormat.getOrElse( + throw new IllegalStateException("fileFormat missing, tagSelfForGpu not called?")) + GpuInsertIntoHiveTable( table = wrapped.table, partition = wrapped.partition, - fileFormat = this.fileFormat.get, + fileFormat = format, query = wrapped.query, overwrite = wrapped.overwrite, ifPartitionNotExists = wrapped.ifPartitionNotExists, @@ -326,8 +327,10 @@ case class GpuInsertIntoHiveTable( if (!fs.delete(path, true)) { throw RapidsHiveErrors.cannotRemovePartitionDirError(path) } - // Don't let Hive do overwrite operation since it is slower. - doHiveOverwrite = false + // Don't let Hive do overwrite operation since it is slower. But still give a + // chance to forcely override this for some customized cases when this + // operation is optimized. + doHiveOverwrite = hadoopConf.getBoolean("hive.movetask.enable.dir.move", false) } } } diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala index 28b8033389a..c8d76f85e5c 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala @@ -47,7 +47,7 @@ import org.apache.spark.sql.execution.command.CommandUtils import org.apache.spark.sql.hive.HiveExternalCatalog import org.apache.spark.sql.hive.client.HiveClientImpl import org.apache.spark.sql.hive.execution.InsertIntoHiveTable -import org.apache.spark.sql.hive.rapids.{GpuHiveTextFileFormat, GpuSaveAsHiveFile, RapidsHiveErrors} +import org.apache.spark.sql.hive.rapids.{GpuHiveFileFormat, GpuSaveAsHiveFile, RapidsHiveErrors} import org.apache.spark.sql.vectorized.ColumnarBatch final class GpuInsertIntoHiveTableMeta(cmd: InsertIntoHiveTable, @@ -59,16 +59,17 @@ final class GpuInsertIntoHiveTableMeta(cmd: InsertIntoHiveTable, private var fileFormat: Option[ColumnarFileFormat] = None override def tagSelfForGpuInternal(): Unit = { - // Only Hive delimited text writes are currently supported. - // Check whether that is the format currently in play. - fileFormat = GpuHiveTextFileFormat.tagGpuSupport(this) + fileFormat = GpuHiveFileFormat.tagGpuSupport(this) } override def convertToGpu(): GpuDataWritingCommand = { + val format = fileFormat.getOrElse( + throw new IllegalStateException("fileFormat missing, tagSelfForGpu not called?")) + GpuInsertIntoHiveTable( table = wrapped.table, partition = wrapped.partition, - fileFormat = this.fileFormat.get, + fileFormat = format, query = wrapped.query, overwrite = wrapped.overwrite, ifPartitionNotExists = wrapped.ifPartitionNotExists, @@ -315,8 +316,10 @@ case class GpuInsertIntoHiveTable( if (!fs.delete(path, true)) { throw RapidsHiveErrors.cannotRemovePartitionDirError(path) } - // Don't let Hive do overwrite operation since it is slower. - doHiveOverwrite = false + // Don't let Hive do overwrite operation since it is slower. But still give a + // chance to forcely override this for some customized cases when this + // operation is optimized. + doHiveOverwrite = hadoopConf.getBoolean("hive.movetask.enable.dir.move", false) } } } From 822ad9bc5a10288e2a1a7af54d9c6f61f9d6e151 Mon Sep 17 00:00:00 2001 From: MithunR Date: Fri, 31 May 2024 00:19:35 -0700 Subject: [PATCH 10/79] [Spark 4.0] Account for `PartitionedFileUtil.splitFiles` signature change. (#10857) * Account for PartitionedFileUtil.splitFiles signature change. Fixes #10299. In Apache Spark 4.0, the signature of `PartitionedFileUtil.splitFiles` was changed to remove unused parameters (apache/spark@eabea643c74). This causes the Spark RAPIDS plugin build to break with Spark 4.0. This commit introduces a shim to account for the signature change. Signed-off-by: MithunR * Common base for PartitionFileUtilsShims. Signed-off-by: MithunR * Reusing existing PartitionedFileUtilsShims. * More refactor, for pre-3.5 compile. * Updated Copyright date. * Fixed style error. * Re-fixed the copyright year. * Added missing import. --------- Signed-off-by: MithunR --- .../shims/PartitionedFileUtilsShim.scala | 22 +-------- .../shims/PartitionedFileUtilsShimBase.scala | 45 +++++++++++++++++++ .../shims/PartitionedFileUtilsShim.scala | 16 ++++++- .../execution/rapids/shims/SplitFiles.scala | 6 +-- .../shims/PartitionedFileUtilsShim.scala | 38 ++++++++++++++++ .../shims/PartitionedFileUtilsShim.scala | 40 +++++++++++++++++ 6 files changed, 141 insertions(+), 26 deletions(-) create mode 100644 sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShimBase.scala create mode 100644 sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala create mode 100644 sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala index ca2fa215892..62fe32ae8db 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala @@ -19,27 +19,7 @@ {"spark": "341"} {"spark": "342"} {"spark": "343"} -{"spark": "350"} -{"spark": "351"} -{"spark": "400"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims -import org.apache.spark.paths.SparkPath -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.execution.datasources.PartitionedFile - -object PartitionedFileUtilsShim { - // Wrapper for case class constructor so Java code can access - // the default values across Spark versions. - def newPartitionedFile( - partitionValues: InternalRow, - filePath: String, - start: Long, - length: Long): PartitionedFile = PartitionedFile(partitionValues, - SparkPath.fromPathString(filePath), start, length) - - def withNewLocations(pf: PartitionedFile, locations: Seq[String]): PartitionedFile = { - pf.copy(locations = locations.toArray) - } -} +object PartitionedFileUtilsShim extends PartitionedFileUtilsShimBase diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShimBase.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShimBase.scala new file mode 100644 index 00000000000..a94c76dc083 --- /dev/null +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShimBase.scala @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "340"} +{"spark": "341"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import org.apache.spark.paths.SparkPath +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.execution.datasources.PartitionedFile + +trait PartitionedFileUtilsShimBase { + + // Wrapper for case class constructor so Java code can access + // the default values across Spark versions. + def newPartitionedFile(partitionValues: InternalRow, + filePath: String, + start: Long, + length: Long): PartitionedFile = PartitionedFile(partitionValues, + SparkPath.fromPathString(filePath), start, length) + + def withNewLocations(pf: PartitionedFile, locations: Seq[String]): PartitionedFile = { + pf.copy(locations = locations.toArray) + } +} diff --git a/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala b/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala index 249502f1b49..0f1bdafde7a 100644 --- a/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala +++ b/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,8 +20,10 @@ spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims import org.apache.spark.paths.SparkPath +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.execution.datasources.PartitionedFile +import org.apache.spark.sql.execution.PartitionedFileUtil +import org.apache.spark.sql.execution.datasources.{FileStatusWithMetadata, PartitionedFile} object PartitionedFileUtilsShim { // Wrapper for case class constructor so Java code can access @@ -37,4 +39,14 @@ object PartitionedFileUtilsShim { def withNewLocations(pf: PartitionedFile, locations: Seq[String]): PartitionedFile = { pf.copy(locations = locations) } + + // In Spark 4.0, PartitionedFileUtil.splitFiles lost its `sparkSession` parameter. + // This pre-Spark-4.0 shim keeps the `sparkSession` parameter. + def splitFiles(sparkSession: SparkSession, + file: FileStatusWithMetadata, + isSplitable: Boolean, + maxSplitBytes: Long, + partitionValues: InternalRow): Seq[PartitionedFile] = { + PartitionedFileUtil.splitFiles(sparkSession, file, isSplitable, maxSplitBytes, partitionValues) + } } diff --git a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/execution/rapids/shims/SplitFiles.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/execution/rapids/shims/SplitFiles.scala index 3b94d5a5201..1934cb6af9f 100644 --- a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/execution/rapids/shims/SplitFiles.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/execution/rapids/shims/SplitFiles.scala @@ -23,12 +23,12 @@ spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.execution.rapids.shims +import com.nvidia.spark.rapids.shims.PartitionedFileUtilsShim import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.io.compress.{CompressionCodecFactory, SplittableCompressionCodec} import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.execution.PartitionedFileUtil import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory, PartitionedFile} trait SplitFiles { @@ -49,7 +49,7 @@ trait SplitFiles { selectedPartitions.flatMap { partition => partition.files.flatMap { f => - PartitionedFileUtil.splitFiles( + PartitionedFileUtilsShim.splitFiles( sparkSession, f, isSplitable = canBeSplit(f.getPath, hadoopConf), @@ -71,7 +71,7 @@ trait SplitFiles { val filePath = file.getPath val isSplitable = relation.fileFormat.isSplitable( relation.sparkSession, relation.options, filePath) - PartitionedFileUtil.splitFiles( + PartitionedFileUtilsShim.splitFiles( sparkSession = relation.sparkSession, file = file, isSplitable = isSplitable, diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala new file mode 100644 index 00000000000..71ad5ae1a0f --- /dev/null +++ b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "350"} +{"spark": "351"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.execution.PartitionedFileUtil +import org.apache.spark.sql.execution.datasources.{FileStatusWithMetadata, PartitionedFile} + +object PartitionedFileUtilsShim extends PartitionedFileUtilsShimBase { + // In Spark 4.0, PartitionedFileUtil.splitFiles lost its `sparkSession` parameter. + // This pre-Spark-4.0 shim keeps the `sparkSession` parameter. + def splitFiles(sparkSession: SparkSession, + file: FileStatusWithMetadata, + isSplitable: Boolean, + maxSplitBytes: Long, + partitionValues: InternalRow): Seq[PartitionedFile] = { + PartitionedFileUtil.splitFiles(sparkSession, file, isSplitable, maxSplitBytes, partitionValues) + } +} diff --git a/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala new file mode 100644 index 00000000000..de8e98962a7 --- /dev/null +++ b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.execution.PartitionedFileUtil +import org.apache.spark.sql.execution.datasources.{FileStatusWithMetadata, PartitionedFile} + +object PartitionedFileUtilsShim extends PartitionedFileUtilsShimBase { + + // In Spark 4.0, PartitionedFileUtil.splitFiles lost its `sparkSession` parameter. + // This Spark-4.0+ shim ignores the `sparkSession` parameter. + def splitFiles(sparkSession: SparkSession, + file: FileStatusWithMetadata, + isSplitable: Boolean, + maxSplitBytes: Long, + partitionValues: InternalRow): Seq[PartitionedFile] = { + PartitionedFileUtil.splitFiles(file, isSplitable, maxSplitBytes, partitionValues) + } + +} // object PartitionFileUtilsShim; From 2a86bb5d42495418815b8322e946d02ff62aa145 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Fri, 31 May 2024 16:11:36 +0800 Subject: [PATCH 11/79] Change dependency version to 24.08.0-SNAPSHOT (#10949) To fix: https://github.com/NVIDIA/spark-rapids/issues/10867 Change rapids private and jni dependency version to 24.08.0-SNAPSHOT Signed-off-by: Tim Liu --- jenkins/databricks/init_cudf_udf.sh | 2 +- jenkins/version-def.sh | 2 +- pom.xml | 5 ++--- scala2.13/pom.xml | 5 ++--- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/jenkins/databricks/init_cudf_udf.sh b/jenkins/databricks/init_cudf_udf.sh index d5c440bfbb2..3c3e73ab582 100755 --- a/jenkins/databricks/init_cudf_udf.sh +++ b/jenkins/databricks/init_cudf_udf.sh @@ -20,7 +20,7 @@ set -ex -CUDF_VER=${CUDF_VER:-24.06} # TODO: https://github.com/NVIDIA/spark-rapids/issues/ +CUDF_VER=${CUDF_VER:-24.08} CUDA_VER=${CUDA_VER:-11.8} # Need to explicitly add conda into PATH environment, to activate conda environment. diff --git a/jenkins/version-def.sh b/jenkins/version-def.sh index d3c01e1eba4..dbad6d6fd94 100755 --- a/jenkins/version-def.sh +++ b/jenkins/version-def.sh @@ -27,7 +27,7 @@ done IFS=$PRE_IFS -CUDF_VER=${CUDF_VER:-"24.06.0-SNAPSHOT"} # TODO: https://github.com/NVIDIA/spark-rapids/issues/ +CUDF_VER=${CUDF_VER:-"24.08.0-SNAPSHOT"} CUDA_CLASSIFIER=${CUDA_CLASSIFIER:-"cuda11"} CLASSIFIER=${CLASSIFIER:-"$CUDA_CLASSIFIER"} # default as CUDA_CLASSIFIER for compatibility PROJECT_VER=${PROJECT_VER:-"24.08.0-SNAPSHOT"} diff --git a/pom.xml b/pom.xml index df010a7589e..38f7e8c3812 100644 --- a/pom.xml +++ b/pom.xml @@ -719,9 +719,8 @@ spark${buildver} cuda11 ${cuda.version} - - 24.06.0-SNAPSHOT - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT + 24.08.0-SNAPSHOT 2.12 2.8.0 incremental diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index 711872e8d54..a6d7de172fe 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -719,9 +719,8 @@ spark${buildver} cuda11 ${cuda.version} - - 24.06.0-SNAPSHOT - 24.06.0-SNAPSHOT + 24.08.0-SNAPSHOT + 24.08.0-SNAPSHOT 2.13 2.8.0 incremental From 2977c14894e4edea97e9ba08746432e7c875c47e Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Fri, 31 May 2024 16:27:12 -0700 Subject: [PATCH 12/79] Add Support for Renaming of PythonMapInArrow [databricks] (#10931) * Add support for the renaming of PythonMapInArrow to MapInArrow * Signing off Signed-off-by: Raza Jafri * Removed the unnecessary base class from 400 * addressed review comments --------- Signed-off-by: Raza Jafri --- .../shims/GpuPythonMapInArrowExec.scala | 1 - .../shims/PythonMapInArrowExecShims.scala | 1 - .../shims/GpuPythonMapInArrowExecMeta.scala | 1 - .../rapids/shims/MapInArrowExecShims.scala | 40 ++++++++ .../rapids/shims/GpuMapInArrowExecMeta.scala | 94 +++++++++++++++++++ 5 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/MapInArrowExecShims.scala create mode 100644 sql-plugin/src/main/spark400/scala/org/apache/spark/sql/rapids/shims/GpuMapInArrowExecMeta.scala diff --git a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/GpuPythonMapInArrowExec.scala b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/GpuPythonMapInArrowExec.scala index 768261cbc89..5118c21ff2e 100644 --- a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/GpuPythonMapInArrowExec.scala +++ b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/GpuPythonMapInArrowExec.scala @@ -31,7 +31,6 @@ {"spark": "343"} {"spark": "350"} {"spark": "351"} -{"spark": "400"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/PythonMapInArrowExecShims.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/PythonMapInArrowExecShims.scala index 833767558c6..8f9bc5c1573 100644 --- a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/PythonMapInArrowExecShims.scala +++ b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/PythonMapInArrowExecShims.scala @@ -17,7 +17,6 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} {"spark": "351"} -{"spark": "400"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims diff --git a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/shims/GpuPythonMapInArrowExecMeta.scala b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/shims/GpuPythonMapInArrowExecMeta.scala index a08211f3795..c27f4824c4a 100644 --- a/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/shims/GpuPythonMapInArrowExecMeta.scala +++ b/sql-plugin/src/main/spark350/scala/org/apache/spark/sql/rapids/shims/GpuPythonMapInArrowExecMeta.scala @@ -17,7 +17,6 @@ /*** spark-rapids-shim-json-lines {"spark": "350"} {"spark": "351"} -{"spark": "400"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.shims diff --git a/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/MapInArrowExecShims.scala b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/MapInArrowExecShims.scala new file mode 100644 index 00000000000..4a1998fa88d --- /dev/null +++ b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/MapInArrowExecShims.scala @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import com.nvidia.spark.rapids._ + +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.python.MapInArrowExec +import org.apache.spark.sql.rapids.shims.GpuMapInArrowExecMeta + +object PythonMapInArrowExecShims { + + def execs: Map[Class[_ <: SparkPlan], ExecRule[_ <: SparkPlan]] = Seq( + GpuOverrides.exec[MapInArrowExec]( + "The backend for Map Arrow Iterator UDF. Accelerates the data transfer between the" + + " Java process and the Python process. It also supports scheduling GPU resources" + + " for the Python process when enabled.", + ExecChecks((TypeSig.commonCudfTypes + TypeSig.ARRAY + TypeSig.STRUCT).nested(), + TypeSig.all), + (mapPy, conf, p, r) => new GpuMapInArrowExecMeta(mapPy, conf, p, r)) + ).map(r => (r.getClassFor.asSubclass(classOf[SparkPlan]), r)).toMap + +} \ No newline at end of file diff --git a/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/rapids/shims/GpuMapInArrowExecMeta.scala b/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/rapids/shims/GpuMapInArrowExecMeta.scala new file mode 100644 index 00000000000..f7010099813 --- /dev/null +++ b/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/rapids/shims/GpuMapInArrowExecMeta.scala @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims + +import com.nvidia.spark.rapids._ + +import org.apache.spark.api.python.PythonEvalType +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, PythonUDF} +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.python.MapInArrowExec +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.rapids.execution.TrampolineUtil +import org.apache.spark.sql.rapids.execution.python.GpuMapInBatchExec +import org.apache.spark.sql.types.{BinaryType, StringType} + +class GpuMapInArrowExecMeta( + mapArrow: MapInArrowExec, + conf: RapidsConf, + parent: Option[RapidsMeta[_, _, _]], + rule: DataFromReplacementRule) + extends SparkPlanMeta[MapInArrowExec](mapArrow, conf, parent, rule) { + override def replaceMessage: String = "partially run on GPU" + + override def noReplacementPossibleMessage(reasons: String): String = + s"cannot run even partially on the GPU because $reasons" + + protected val udf: BaseExprMeta[PythonUDF] = GpuOverrides.wrapExpr( + mapArrow.func.asInstanceOf[PythonUDF], conf, Some(this)) + protected val resultAttrs: Seq[BaseExprMeta[Attribute]] = + mapArrow.output.map(GpuOverrides.wrapExpr(_, conf, Some(this))) + + override val childExprs: Seq[BaseExprMeta[_]] = resultAttrs :+ udf + + override def tagPlanForGpu(): Unit = { + super.tagPlanForGpu() + if (SQLConf.get.getConf(SQLConf.ARROW_EXECUTION_USE_LARGE_VAR_TYPES)) { + + val inputTypes = mapArrow.child.schema.fields.map(_.dataType) + val outputTypes = mapArrow.output.map(_.dataType) + + val hasStringOrBinaryTypes = (inputTypes ++ outputTypes).exists(dataType => + TrampolineUtil.dataTypeExistsRecursively(dataType, + dt => dt == StringType || dt == BinaryType)) + + if (hasStringOrBinaryTypes) { + willNotWorkOnGpu(s"${SQLConf.ARROW_EXECUTION_USE_LARGE_VAR_TYPES.key} is " + + s"enabled and the schema contains string or binary types. This is not " + + s"supported on the GPU.") + } + } + } + + override def convertToGpu(): GpuExec = + GpuMapInArrowExec( + udf.convertToGpu(), + resultAttrs.map(_.convertToGpu()).asInstanceOf[Seq[Attribute]], + childPlans.head.convertIfNeeded(), + isBarrier = mapArrow.isBarrier, + ) +} + +/* + * A relation produced by applying a function that takes an iterator of PyArrow's record + * batches and outputs an iterator of PyArrow's record batches. + * + * This GpuMapInPandasExec aims at accelerating the data transfer between + * JVM and Python, and scheduling GPU resources for its Python processes. + * + */ +case class GpuMapInArrowExec( + func: Expression, + output: Seq[Attribute], + child: SparkPlan, + override val isBarrier: Boolean) extends GpuMapInBatchExec { + + override protected val pythonEvalType: Int = PythonEvalType.SQL_MAP_ARROW_ITER_UDF +} From 1be42d48ae64595e8e35ce54b332f54e73d4d8a7 Mon Sep 17 00:00:00 2001 From: Liangcai Li Date: Sat, 1 Jun 2024 08:09:32 +0800 Subject: [PATCH 13/79] fix build errors for 4.0 shim (#10952) Signed-off-by: Firestarman --- .../python/GpuAggregateInPandasExec.scala | 8 +-- .../python/GpuArrowEvalPythonExec.scala | 9 +-- .../GpuFlatMapCoGroupsInPandasExec.scala | 6 +- .../python/GpuFlatMapGroupsInPandasExec.scala | 4 +- .../execution/python/GpuMapInBatchExec.scala | 4 +- .../execution/python/GpuPythonHelper.scala | 7 ++- .../execution/python/GpuPythonUDF.scala | 6 +- .../python/GpuWindowInPandasExecBase.scala | 12 ++-- .../python/shims/GpuArrowPythonRunner.scala | 9 ++- .../shims/GpuCoGroupedArrowPythonRunner.scala | 7 +-- .../shims/GpuGroupedPythonRunnerFactory.scala | 2 +- .../python/shims/WritePythonUDFUtils.scala | 59 +++++++++++++++++++ .../shims/GpuGroupUDFArrowPythonRunner.scala | 9 ++- .../shims/GpuGroupedPythonRunnerFactory.scala | 2 +- .../python/shims/GpuArrowPythonRunner.scala | 9 ++- .../shims/GpuCoGroupedArrowPythonRunner.scala | 7 +-- .../shims/GpuGroupUDFArrowPythonRunner.scala | 9 ++- .../shims/GpuGroupedPythonRunnerFactory.scala | 2 +- .../python/shims/WritePythonUDFUtils.scala | 35 +++++++++++ 19 files changed, 148 insertions(+), 58 deletions(-) create mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala create mode 100644 sql-plugin/src/main/spark400/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuAggregateInPandasExec.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuAggregateInPandasExec.scala index bc2f30dff2f..639a39bcd38 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuAggregateInPandasExec.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuAggregateInPandasExec.scala @@ -75,15 +75,15 @@ case class GpuAggregateInPandasExec( } private def collectFunctions(udf: GpuPythonFunction): - (ChainedPythonFunctions, Seq[Expression]) = { + ((ChainedPythonFunctions, Long), Seq[Expression]) = { udf.children match { case Seq(u: GpuPythonFunction) => - val (chained, children) = collectFunctions(u) - (ChainedPythonFunctions(chained.funcs ++ Seq(udf.func)), children) + val ((chained, _), children) = collectFunctions(u) + ((ChainedPythonFunctions(chained.funcs ++ Seq(udf.func)), udf.resultId.id), children) case children => // There should not be any other UDFs, or the children can't be evaluated directly. assert(children.forall(_.find(_.isInstanceOf[GpuPythonFunction]).isEmpty)) - (ChainedPythonFunctions(Seq(udf.func)), udf.children) + ((ChainedPythonFunctions(Seq(udf.func)), udf.resultId.id), udf.children) } } diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuArrowEvalPythonExec.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuArrowEvalPythonExec.scala index 182d7d1b6c6..c99d0403ed0 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuArrowEvalPythonExec.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuArrowEvalPythonExec.scala @@ -362,15 +362,16 @@ case class GpuArrowEvalPythonExec( override def producedAttributes: AttributeSet = AttributeSet(resultAttrs) - private def collectFunctions(udf: GpuPythonUDF): (ChainedPythonFunctions, Seq[Expression]) = { + private def collectFunctions( + udf: GpuPythonUDF): ((ChainedPythonFunctions, Long), Seq[Expression]) = { udf.children match { case Seq(u: GpuPythonUDF) => - val (chained, children) = collectFunctions(u) - (ChainedPythonFunctions(chained.funcs ++ Seq(udf.func)), children) + val ((chained, _), children) = collectFunctions(u) + ((ChainedPythonFunctions(chained.funcs ++ Seq(udf.func)), udf.resultId.id), children) case children => // There should not be any other UDFs, or the children can't be evaluated directly. assert(children.forall(_.find(_.isInstanceOf[GpuPythonUDF]).isEmpty)) - (ChainedPythonFunctions(Seq(udf.func)), udf.children) + ((ChainedPythonFunctions(Seq(udf.func)), udf.resultId.id), udf.children) } } diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuFlatMapCoGroupsInPandasExec.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuFlatMapCoGroupsInPandasExec.scala index b8fa3c1ab69..2e90765e40e 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuFlatMapCoGroupsInPandasExec.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuFlatMapCoGroupsInPandasExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -107,8 +107,8 @@ case class GpuFlatMapCoGroupsInPandasExec( private val sessionLocalTimeZone = conf.sessionLocalTimeZone private val pythonRunnerConf = ArrowUtilsShim.getPythonRunnerConfMap(conf) - private val pandasFunction = udf.asInstanceOf[GpuPythonUDF].func - private val chainedFunc = Seq(ChainedPythonFunctions(Seq(pandasFunction))) + private val pyUDF = udf.asInstanceOf[GpuPythonUDF] + private val chainedFunc = Seq((ChainedPythonFunctions(Seq(pyUDF.func)), pyUDF.resultId.id)) override def producedAttributes: AttributeSet = AttributeSet(output) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuFlatMapGroupsInPandasExec.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuFlatMapGroupsInPandasExec.scala index 4a24a449b24..f1596ae7a74 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuFlatMapGroupsInPandasExec.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuFlatMapGroupsInPandasExec.scala @@ -98,7 +98,7 @@ case class GpuFlatMapGroupsInPandasExec( override def requiredChildOrdering: Seq[Seq[SortOrder]] = Seq(groupingAttributes.map(SortOrder(_, Ascending))) - private val pandasFunction = func.asInstanceOf[GpuPythonUDF].func + private val udf = func.asInstanceOf[GpuPythonUDF] // One batch as input to keep the integrity for each group override def childrenCoalesceGoal: Seq[CoalesceGoal] = Seq(RequireSingleBatch) @@ -111,7 +111,7 @@ case class GpuFlatMapGroupsInPandasExec( val (mNumInputRows, mNumInputBatches, mNumOutputRows, mNumOutputBatches) = commonGpuMetrics() lazy val isPythonOnGpuEnabled = GpuPythonHelper.isPythonOnGpuEnabled(conf) - val chainedFunc = Seq(ChainedPythonFunctions(Seq(pandasFunction))) + val chainedFunc = Seq((ChainedPythonFunctions(Seq(udf.func)), udf.resultId.id)) val localOutput = output val localChildOutput = child.output // Python wraps the resulting columns in a single struct column. diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuMapInBatchExec.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuMapInBatchExec.scala index 4d41cd32e4f..57c1c7f7114 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuMapInBatchExec.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuMapInBatchExec.scala @@ -46,7 +46,7 @@ trait GpuMapInBatchExec extends ShimUnaryExecNode with GpuPythonExecBase { protected val isBarrier: Boolean - private val pandasFunction = func.asInstanceOf[GpuPythonUDF].func + private val udf = func.asInstanceOf[GpuPythonUDF] override def producedAttributes: AttributeSet = AttributeSet(output) @@ -58,7 +58,7 @@ trait GpuMapInBatchExec extends ShimUnaryExecNode with GpuPythonExecBase { val (numInputRows, numInputBatches, numOutputRows, numOutputBatches) = commonGpuMetrics() val pyInputTypes = child.schema - val chainedFunc = Seq(ChainedPythonFunctions(Seq(pandasFunction))) + val chainedFunc = Seq((ChainedPythonFunctions(Seq(udf.func)), udf.resultId.id)) val sessionLocalTimeZone = conf.sessionLocalTimeZone val pythonRunnerConf = ArrowUtilsShim.getPythonRunnerConfMap(conf) val isPythonOnGpuEnabled = GpuPythonHelper.isPythonOnGpuEnabled(conf) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuPythonHelper.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuPythonHelper.scala index 451ae401891..8564018ad3b 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuPythonHelper.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuPythonHelper.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -86,11 +86,12 @@ object GpuPythonHelper extends Logging { } // Called in each task at the executor side - def injectGpuInfo(funcs: Seq[ChainedPythonFunctions], isPythonOnGpuEnabled: Boolean): Unit = { + def injectGpuInfo(funcs: Seq[(ChainedPythonFunctions, Long)], + isPythonOnGpuEnabled: Boolean): Unit = { // Insert GPU related env(s) into `envVars` for all the PythonFunction(s). // Yes `PythonRunner` will only use the first one, but just make sure it will // take effect no matter the order changes or not. - funcs.foreach(_.funcs.foreach { pyF => + funcs.foreach(_._1.funcs.foreach { pyF => pyF.envVars.put("CUDA_VISIBLE_DEVICES", gpuId) pyF.envVars.put("RAPIDS_PYTHON_ENABLED", isPythonOnGpuEnabled.toString) pyF.envVars.put("RAPIDS_UVM_ENABLED", isPythonUvmEnabled) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuPythonUDF.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuPythonUDF.scala index 6cb955a6db8..04367d9f29f 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuPythonUDF.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuPythonUDF.scala @@ -64,7 +64,7 @@ abstract class GpuPythonFunction( children: Seq[Expression], evalType: Int, udfDeterministic: Boolean, - resultId: ExprId = NamedExpression.newExprId) + val resultId: ExprId = NamedExpression.newExprId) extends Expression with GpuUnevaluable with NonSQLExpression with UserDefinedExpression with GpuAggregateWindowFunction with Serializable { @@ -94,7 +94,7 @@ case class GpuPythonUDF( children: Seq[Expression], evalType: Int, udfDeterministic: Boolean, - resultId: ExprId = NamedExpression.newExprId) + override val resultId: ExprId = NamedExpression.newExprId) extends GpuPythonFunction(name, func, dataType, children, evalType, udfDeterministic, resultId) { override lazy val canonicalized: Expression = { val canonicalizedChildren = children.map(_.canonicalized) @@ -110,7 +110,7 @@ case class GpuPythonUDAF( children: Seq[Expression], evalType: Int, udfDeterministic: Boolean, - resultId: ExprId = NamedExpression.newExprId) + override val resultId: ExprId = NamedExpression.newExprId) extends GpuPythonFunction(name, func, dataType, children, evalType, udfDeterministic, resultId) with GpuAggregateFunction { override lazy val canonicalized: Expression = { diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuWindowInPandasExecBase.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuWindowInPandasExecBase.scala index 3bc91cd6338..fcf9570a9f7 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuWindowInPandasExecBase.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuWindowInPandasExecBase.scala @@ -235,16 +235,16 @@ trait GpuWindowInPandasExecBase extends ShimUnaryExecNode with GpuPythonExecBase protected val windowBoundTypeConf = "pandas_window_bound_types" - protected def collectFunctions(udf: GpuPythonFunction): - (ChainedPythonFunctions, Seq[Expression]) = { + protected def collectFunctions( + udf: GpuPythonFunction): ((ChainedPythonFunctions, Long), Seq[Expression]) = { udf.children match { case Seq(u: GpuPythonFunction) => - val (chained, children) = collectFunctions(u) - (ChainedPythonFunctions(chained.funcs ++ Seq(udf.func)), children) + val ((chained, _), children) = collectFunctions(u) + ((ChainedPythonFunctions(chained.funcs ++ Seq(udf.func)), udf.resultId.id), children) case children => // There should not be any other UDFs, or the children can't be evaluated directly. assert(children.forall(_.find(_.isInstanceOf[GpuPythonFunction]).isEmpty)) - (ChainedPythonFunctions(Seq(udf.func)), udf.children) + ((ChainedPythonFunctions(Seq(udf.func)), udf.resultId.id), udf.children) } } @@ -396,7 +396,7 @@ trait GpuWindowInPandasExecBase extends ShimUnaryExecNode with GpuPythonExecBase } }.toArray val dataCVs = GpuColumnVector.extractColumns(batch) - new ColumnarBatch(boundsCVs ++ dataCVs.map(_.incRefCount()), numRows) + new ColumnarBatch((boundsCVs ++ dataCVs.map(_.incRefCount())).toArray, numRows) } override protected def internalDoExecuteColumnar(): RDD[ColumnarBatch] = { diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala index 761d84b4667..977c755712a 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala @@ -49,7 +49,6 @@ import com.nvidia.spark.rapids.GpuSemaphore import org.apache.spark.{SparkEnv, TaskContext} import org.apache.spark.api.python.ChainedPythonFunctions -import org.apache.spark.sql.execution.python.PythonUDFRunner import org.apache.spark.sql.rapids.execution.python.{GpuArrowPythonWriter, GpuPythonRunnerCommon} import org.apache.spark.sql.rapids.shims.ArrowUtilsShim import org.apache.spark.sql.types.StructType @@ -60,7 +59,7 @@ import org.apache.spark.util.Utils * Similar to `PythonUDFRunner`, but exchange data with Python worker via Arrow stream. */ class GpuArrowPythonRunner( - funcs: Seq[ChainedPythonFunctions], + funcs: Seq[(ChainedPythonFunctions, Long)], evalType: Int, argOffsets: Array[Array[Int]], pythonInSchema: StructType, @@ -69,8 +68,8 @@ class GpuArrowPythonRunner( maxBatchSize: Long, override val pythonOutSchema: StructType, jobArtifactUUID: Option[String] = None) - extends GpuBasePythonRunner[ColumnarBatch](funcs, evalType, argOffsets, jobArtifactUUID) - with GpuArrowPythonOutput with GpuPythonRunnerCommon { + extends GpuBasePythonRunner[ColumnarBatch](funcs.map(_._1), evalType, argOffsets, + jobArtifactUUID) with GpuArrowPythonOutput with GpuPythonRunnerCommon { protected override def newWriterThread( env: SparkEnv, @@ -82,7 +81,7 @@ class GpuArrowPythonRunner( val arrowWriter = new GpuArrowPythonWriter(pythonInSchema, maxBatchSize) { override protected def writeUDFs(dataOut: DataOutputStream): Unit = { - PythonUDFRunner.writeUDFs(dataOut, funcs, argOffsets) + WritePythonUDFUtils.writeUDFs(dataOut, funcs, argOffsets) } } val isInputNonEmpty = inputIterator.nonEmpty diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala index adb28725ba1..68112676a2b 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala @@ -50,7 +50,6 @@ import com.nvidia.spark.rapids.GpuSemaphore import org.apache.spark.{SparkEnv, TaskContext} import org.apache.spark.api.python.{ChainedPythonFunctions, PythonRDD} -import org.apache.spark.sql.execution.python.PythonUDFRunner import org.apache.spark.sql.rapids.execution.python.{GpuArrowWriter, GpuPythonRunnerCommon} import org.apache.spark.sql.types.StructType import org.apache.spark.sql.vectorized.ColumnarBatch @@ -63,7 +62,7 @@ import org.apache.spark.util.Utils * and receive it back in JVM as batches of single DataFrame. */ class GpuCoGroupedArrowPythonRunner( - funcs: Seq[ChainedPythonFunctions], + funcs: Seq[(ChainedPythonFunctions, Long)], evalType: Int, argOffsets: Array[Array[Int]], leftSchema: StructType, @@ -73,7 +72,7 @@ class GpuCoGroupedArrowPythonRunner( batchSize: Int, override val pythonOutSchema: StructType, jobArtifactUUID: Option[String] = None) - extends GpuBasePythonRunner[(ColumnarBatch, ColumnarBatch)](funcs, evalType, + extends GpuBasePythonRunner[(ColumnarBatch, ColumnarBatch)](funcs.map(_._1), evalType, argOffsets, jobArtifactUUID) with GpuArrowPythonOutput with GpuPythonRunnerCommon { protected override def newWriterThread( @@ -90,7 +89,7 @@ class GpuCoGroupedArrowPythonRunner( PythonRDD.writeUTF(k, dataOut) PythonRDD.writeUTF(v, dataOut) } - PythonUDFRunner.writeUDFs(dataOut, funcs, argOffsets) + WritePythonUDFUtils.writeUDFs(dataOut, funcs, argOffsets) } protected override def writeIteratorToStream(dataOut: DataOutputStream): Unit = { diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala index eba0286e181..9df93a9d11b 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala @@ -48,7 +48,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch case class GpuGroupedPythonRunnerFactory( conf: org.apache.spark.sql.internal.SQLConf, - chainedFunc: Seq[ChainedPythonFunctions], + chainedFunc: Seq[(ChainedPythonFunctions, Long)], argOffsets: Array[Array[Int]], dedupAttrs: StructType, pythonOutputSchema: StructType, diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala new file mode 100644 index 00000000000..aacf972e7e0 --- /dev/null +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "311"} +{"spark": "312"} +{"spark": "313"} +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.execution.python.shims + +import java.io.DataOutputStream + +import org.apache.spark.api.python.ChainedPythonFunctions +import org.apache.spark.sql.execution.python.PythonUDFRunner + +object WritePythonUDFUtils { + def writeUDFs( + dataOut: DataOutputStream, + funcs: Seq[(ChainedPythonFunctions, Long)], + argOffsets: Array[Array[Int]], + profiler: Option[String] = None): Unit = { + PythonUDFRunner.writeUDFs(dataOut, funcs.map(_._1), argOffsets) + } +} diff --git a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala index cb8eef809f3..a6338e7adc5 100644 --- a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala @@ -30,7 +30,6 @@ import com.nvidia.spark.rapids.GpuSemaphore import org.apache.spark.{SparkEnv, TaskContext} import org.apache.spark.api.python._ -import org.apache.spark.sql.execution.python.PythonUDFRunner import org.apache.spark.sql.rapids.execution.python.{GpuArrowPythonWriter, GpuPythonRunnerCommon} import org.apache.spark.sql.types.StructType import org.apache.spark.sql.vectorized.ColumnarBatch @@ -50,7 +49,7 @@ import org.apache.spark.util.Utils * more data being sent. */ class GpuGroupUDFArrowPythonRunner( - funcs: Seq[ChainedPythonFunctions], + funcs: Seq[(ChainedPythonFunctions, Long)], evalType: Int, argOffsets: Array[Array[Int]], pythonInSchema: StructType, @@ -59,8 +58,8 @@ class GpuGroupUDFArrowPythonRunner( maxBatchSize: Long, override val pythonOutSchema: StructType, jobArtifactUUID: Option[String] = None) - extends GpuBasePythonRunner[ColumnarBatch](funcs, evalType, argOffsets, jobArtifactUUID) - with GpuArrowPythonOutput with GpuPythonRunnerCommon { + extends GpuBasePythonRunner[ColumnarBatch](funcs.map(_._1), evalType, argOffsets, + jobArtifactUUID) with GpuArrowPythonOutput with GpuPythonRunnerCommon { protected override def newWriterThread( env: SparkEnv, @@ -72,7 +71,7 @@ class GpuGroupUDFArrowPythonRunner( val arrowWriter = new GpuArrowPythonWriter(pythonInSchema, maxBatchSize) { override protected def writeUDFs(dataOut: DataOutputStream): Unit = { - PythonUDFRunner.writeUDFs(dataOut, funcs, argOffsets) + WritePythonUDFUtils.writeUDFs(dataOut, funcs, argOffsets) } } diff --git a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala index 451de0a2527..313ea6c20a2 100644 --- a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala +++ b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch case class GpuGroupedPythonRunnerFactory( conf: org.apache.spark.sql.internal.SQLConf, - chainedFunc: Seq[ChainedPythonFunctions], + chainedFunc: Seq[(ChainedPythonFunctions, Long)], argOffsets: Array[Array[Int]], dedupAttrs: StructType, pythonOutputSchema: StructType, diff --git a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala index ac58baa2eb7..50c5e280e9c 100644 --- a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala @@ -25,7 +25,6 @@ import com.nvidia.spark.rapids.GpuSemaphore import org.apache.spark.{SparkEnv, TaskContext} import org.apache.spark.api.python._ -import org.apache.spark.sql.execution.python.PythonUDFRunner import org.apache.spark.sql.rapids.execution.python.{GpuArrowPythonWriter, GpuPythonRunnerCommon} import org.apache.spark.sql.rapids.shims.ArrowUtilsShim import org.apache.spark.sql.types.StructType @@ -35,7 +34,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch * Similar to `PythonUDFRunner`, but exchange data with Python worker via Arrow stream. */ class GpuArrowPythonRunner( - funcs: Seq[ChainedPythonFunctions], + funcs: Seq[(ChainedPythonFunctions, Long)], evalType: Int, argOffsets: Array[Array[Int]], pythonInSchema: StructType, @@ -44,8 +43,8 @@ class GpuArrowPythonRunner( maxBatchSize: Long, override val pythonOutSchema: StructType, jobArtifactUUID: Option[String] = None) - extends GpuBasePythonRunner[ColumnarBatch](funcs, evalType, argOffsets, jobArtifactUUID) - with GpuArrowPythonOutput with GpuPythonRunnerCommon { + extends GpuBasePythonRunner[ColumnarBatch](funcs.map(_._1), evalType, argOffsets, + jobArtifactUUID) with GpuArrowPythonOutput with GpuPythonRunnerCommon { protected override def newWriter( env: SparkEnv, @@ -57,7 +56,7 @@ class GpuArrowPythonRunner( val arrowWriter = new GpuArrowPythonWriter(pythonInSchema, maxBatchSize) { override protected def writeUDFs(dataOut: DataOutputStream): Unit = { - PythonUDFRunner.writeUDFs(dataOut, funcs, argOffsets) + WritePythonUDFUtils.writeUDFs(dataOut, funcs, argOffsets) } } val isInputNonEmpty = inputIterator.nonEmpty diff --git a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala index aad1eb52c02..0317a89009e 100644 --- a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala @@ -27,7 +27,6 @@ import com.nvidia.spark.rapids.GpuSemaphore import org.apache.spark.{SparkEnv, TaskContext} import org.apache.spark.api.python.{ChainedPythonFunctions, PythonRDD, PythonWorker} -import org.apache.spark.sql.execution.python.PythonUDFRunner import org.apache.spark.sql.rapids.execution.python.{GpuArrowWriter, GpuPythonRunnerCommon} import org.apache.spark.sql.types.StructType import org.apache.spark.sql.vectorized.ColumnarBatch @@ -39,7 +38,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch * and receive it back in JVM as batches of single DataFrame. */ class GpuCoGroupedArrowPythonRunner( - funcs: Seq[ChainedPythonFunctions], + funcs: Seq[(ChainedPythonFunctions, Long)], evalType: Int, argOffsets: Array[Array[Int]], leftSchema: StructType, @@ -49,7 +48,7 @@ class GpuCoGroupedArrowPythonRunner( batchSize: Int, override val pythonOutSchema: StructType, jobArtifactUUID: Option[String] = None) - extends GpuBasePythonRunner[(ColumnarBatch, ColumnarBatch)](funcs, evalType, + extends GpuBasePythonRunner[(ColumnarBatch, ColumnarBatch)](funcs.map(_._1), evalType, argOffsets, jobArtifactUUID) with GpuArrowPythonOutput with GpuPythonRunnerCommon { protected override def newWriter( @@ -67,7 +66,7 @@ class GpuCoGroupedArrowPythonRunner( PythonRDD.writeUTF(k, dataOut) PythonRDD.writeUTF(v, dataOut) } - PythonUDFRunner.writeUDFs(dataOut, funcs, argOffsets) + WritePythonUDFUtils.writeUDFs(dataOut, funcs, argOffsets) } override def writeNextInputToStream(dataOut: DataOutputStream): Boolean = { diff --git a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala index 4393c8b7057..42c6178ff83 100644 --- a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala @@ -28,7 +28,6 @@ import com.nvidia.spark.rapids.GpuSemaphore import org.apache.spark.{SparkEnv, TaskContext} import org.apache.spark.api.python._ -import org.apache.spark.sql.execution.python.PythonUDFRunner import org.apache.spark.sql.rapids.execution.python.{GpuArrowPythonWriter, GpuPythonRunnerCommon} import org.apache.spark.sql.types.StructType import org.apache.spark.sql.vectorized.ColumnarBatch @@ -47,7 +46,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch * more data being sent. */ class GpuGroupUDFArrowPythonRunner( - funcs: Seq[ChainedPythonFunctions], + funcs: Seq[(ChainedPythonFunctions, Long)], evalType: Int, argOffsets: Array[Array[Int]], pythonInSchema: StructType, @@ -56,8 +55,8 @@ class GpuGroupUDFArrowPythonRunner( batchSize: Long, override val pythonOutSchema: StructType, jobArtifactUUID: Option[String] = None) - extends GpuBasePythonRunner[ColumnarBatch](funcs, evalType, argOffsets, jobArtifactUUID) - with GpuArrowPythonOutput with GpuPythonRunnerCommon { + extends GpuBasePythonRunner[ColumnarBatch](funcs.map(_._1), evalType, argOffsets, + jobArtifactUUID) with GpuArrowPythonOutput with GpuPythonRunnerCommon { protected override def newWriter( env: SparkEnv, @@ -69,7 +68,7 @@ class GpuGroupUDFArrowPythonRunner( val arrowWriter = new GpuArrowPythonWriter(pythonInSchema, batchSize) { override protected def writeUDFs(dataOut: DataOutputStream): Unit = { - PythonUDFRunner.writeUDFs(dataOut, funcs, argOffsets) + WritePythonUDFUtils.writeUDFs(dataOut, funcs, argOffsets) } } diff --git a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala index b1dabbf5b5e..63a4289c5b0 100644 --- a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch case class GpuGroupedPythonRunnerFactory( conf: org.apache.spark.sql.internal.SQLConf, - chainedFunc: Seq[ChainedPythonFunctions], + chainedFunc: Seq[(ChainedPythonFunctions, Long)], argOffsets: Array[Array[Int]], dedupAttrs: StructType, pythonOutputSchema: StructType, diff --git a/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala b/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala new file mode 100644 index 00000000000..4650d998fd7 --- /dev/null +++ b/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.execution.python.shims + +import java.io.DataOutputStream + +import org.apache.spark.api.python.ChainedPythonFunctions +import org.apache.spark.sql.execution.python.PythonUDFRunner + +object WritePythonUDFUtils { + def writeUDFs( + dataOut: DataOutputStream, + funcs: Seq[(ChainedPythonFunctions, Long)], + argOffsets: Array[Array[Int]], + profiler: Option[String] = None): Unit = { + PythonUDFRunner.writeUDFs(dataOut, funcs, argOffsets, profiler) + } +} From 5750ace5bcece0aeba1d3b515bb8f70a9f423878 Mon Sep 17 00:00:00 2001 From: Peixin Date: Mon, 3 Jun 2024 15:15:38 +0800 Subject: [PATCH 14/79] Add new blossom-ci allowed user (#10959) Signed-off-by: Peixin Li --- .github/workflows/blossom-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml index b3cbbb6ad14..6f597d6baf3 100644 --- a/.github/workflows/blossom-ci.yml +++ b/.github/workflows/blossom-ci.yml @@ -71,6 +71,7 @@ jobs: liurenjie1024,\ binmahone,\ zpuller,\ + pxLi,\ ', format('{0},', github.actor)) && github.event.comment.body == 'build' steps: - name: Check if comment is issued by authorized person From 8d3a8ced98d2f04b0021fe5759df3984e6340991 Mon Sep 17 00:00:00 2001 From: Peixin Date: Tue, 4 Jun 2024 12:02:24 +0800 Subject: [PATCH 15/79] Add default value for REF of premerge jenkinsfile to avoid bad overwritten [skip ci] (#10966) * DO NOT REVIEW Signed-off-by: Peixin Li * Add default value for REF to avoid overwritten while unexpected manual trigger Signed-off-by: Peixin Li --------- Signed-off-by: Peixin Li --- jenkins/Jenkinsfile-blossom.premerge | 3 ++- jenkins/Jenkinsfile-blossom.premerge-databricks | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/jenkins/Jenkinsfile-blossom.premerge b/jenkins/Jenkinsfile-blossom.premerge index e7bb8af2cdd..fb3a5c2f328 100755 --- a/jenkins/Jenkinsfile-blossom.premerge +++ b/jenkins/Jenkinsfile-blossom.premerge @@ -57,7 +57,8 @@ pipeline { } parameters { - string(name: 'REF', defaultValue: '', + // Put a default value for REF to avoid error when running the pipeline manually + string(name: 'REF', defaultValue: 'main', description: 'Merged commit of specific PR') string(name: 'GITHUB_DATA', defaultValue: '', description: 'Json-formatted github data from upstream blossom-ci') diff --git a/jenkins/Jenkinsfile-blossom.premerge-databricks b/jenkins/Jenkinsfile-blossom.premerge-databricks index a13170f7162..cd872fd9153 100644 --- a/jenkins/Jenkinsfile-blossom.premerge-databricks +++ b/jenkins/Jenkinsfile-blossom.premerge-databricks @@ -46,7 +46,8 @@ pipeline { } parameters { - string(name: 'REF', defaultValue: '', + // Put a default value for REF to avoid error when running the pipeline manually + string(name: 'REF', defaultValue: 'main', description: 'Merged commit of specific PR') string(name: 'GITHUB_DATA', defaultValue: '', description: 'Json-formatted github data from upstream blossom-ci') From 47074062b92f02cfd89f37766aefd949c2095900 Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Tue, 4 Jun 2024 12:30:15 -0700 Subject: [PATCH 16/79] Use ErrorClass to Throw AnalysisException [databricks] (#10830) * AnalysisException child class Signed-off-by: Raza Jafri * Use errorClass for reporting AnalysisException * POM changes Signed-off-by: Raza Jafri * Reuse the RapidsErrorUtils to throw the AnalysisException * Revert "POM changes" This reverts commit 0f765c9b3910aa3c174de709850d9a5fbc4aea89. * Updated copyrights * Added the TrampolineUtil method back to handle cases which don't use errorClass * Add doc to the RapidsAnalysisException * addressed review comments * Fixed imports * Moved the RapidsAnalysisException out of TrampolineUtil * fixed imports * addressed review comments * fixed unused import * Removed the TrampolineUtil method for throwing RapidsAnalysisException --------- Signed-off-by: Raza Jafri --- .../rapids/GpuDataWritingCommandExec.scala | 11 +- .../spark/rapids/GpuRunnableCommandExec.scala | 11 +- .../sql/hive/rapids/RapidsHiveErrors.scala | 7 +- .../spark/sql/rapids/GpuDataSourceBase.scala | 51 +++---- ...GpuInsertIntoHadoopFsRelationCommand.scala | 8 +- .../expressions/GpuRandomExpressions.scala | 6 +- .../sql/rapids/execution/TrampolineUtil.scala | 12 +- .../rapids/shims/ParquetSchemaClipShims.scala | 16 +-- .../rapids/shims/GpuInsertIntoHiveTable.scala | 5 +- .../sql/rapids/shims/RapidsErrorUtils.scala | 4 +- .../rapids/shims/RapidsQueryErrorUtils.scala | 125 +++++++++++++++++ .../rapids/shims/ParquetSchemaClipShims.scala | 13 +- .../sql/rapids/shims/RapidsErrorUtils.scala | 4 +- .../rapids/shims/RapidsQueryErrorUtils.scala | 127 ++++++++++++++++++ .../rapids/shims/ParquetSchemaClipShims.scala | 14 +- .../sql/rapids/shims/RapidsErrorUtils.scala | 4 +- .../sql/rapids/shims/RapidsErrorUtils.scala | 4 +- .../rapids/shims/GpuInsertIntoHiveTable.scala | 5 +- .../sql/rapids/GpuFileFormatWriter.scala | 5 +- ...eDataSourceTableAsSelectCommandShims.scala | 2 +- .../sql/rapids/shims/RapidsErrorUtils.scala | 2 +- .../sql/rapids/shims/RapidsErrorUtils.scala | 4 +- 22 files changed, 342 insertions(+), 98 deletions(-) create mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsQueryErrorUtils.scala create mode 100644 sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsQueryErrorUtils.scala diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuDataWritingCommandExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuDataWritingCommandExec.scala index 5a54d0b2f66..019f9b2e6b0 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuDataWritingCommandExec.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuDataWritingCommandExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ import org.apache.spark.sql.execution.command.DataWritingCommand import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.rapids.GpuWriteJobStatsTracker -import org.apache.spark.sql.rapids.execution.TrampolineUtil +import org.apache.spark.sql.rapids.shims.RapidsErrorUtils import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.SerializableConfiguration @@ -84,10 +84,9 @@ object GpuDataWritingCommand { if (fs.exists(filePath) && fs.getFileStatus(filePath).isDirectory && fs.listStatus(filePath).length != 0) { - TrampolineUtil.throwAnalysisException( - s"CREATE-TABLE-AS-SELECT cannot create table with location to a non-empty directory " + - s"${tablePath} . To allow overwriting the existing non-empty directory, " + - s"set '$allowNonEmptyLocationInCTASKey' to true.") + throw RapidsErrorUtils. + createTableAsSelectWithNonEmptyDirectoryError(tablePath.toString, + allowNonEmptyLocationInCTASKey) } } } diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuRunnableCommandExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuRunnableCommandExec.scala index e3869960fc4..43bd593c0b5 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuRunnableCommandExec.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuRunnableCommandExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ import org.apache.spark.sql.execution.command.RunnableCommand import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.rapids.GpuWriteJobStatsTracker -import org.apache.spark.sql.rapids.execution.TrampolineUtil +import org.apache.spark.sql.rapids.shims.RapidsErrorUtils import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.SerializableConfiguration @@ -82,10 +82,9 @@ object GpuRunnableCommand { if (fs.exists(filePath) && fs.getFileStatus(filePath).isDirectory && fs.listStatus(filePath).length != 0) { - TrampolineUtil.throwAnalysisException( - s"CREATE-TABLE-AS-SELECT cannot create table with location to a non-empty directory " + - s"${tablePath} . To allow overwriting the existing non-empty directory, " + - s"set '$allowNonEmptyLocationInCTASKey' to true.") + throw RapidsErrorUtils. + createTableAsSelectWithNonEmptyDirectoryError(tablePath.toString, + allowNonEmptyLocationInCTASKey) } } } diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/RapidsHiveErrors.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/RapidsHiveErrors.scala index 259a04ec318..40cac90680f 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/RapidsHiveErrors.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/RapidsHiveErrors.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,9 +19,9 @@ package org.apache.spark.sql.hive.rapids import org.apache.hadoop.fs.Path import org.apache.spark.SparkException -import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.Literal +import org.apache.spark.sql.rapids.shims.RapidsErrorUtils import org.apache.spark.sql.types.{DataType, DoubleType, FloatType, StringType} object RapidsHiveErrors { @@ -53,8 +53,7 @@ object RapidsHiveErrors { } def cannotResolveAttributeError(name: String, outputStr: String): Throwable = { - new AnalysisException( - s"Unable to resolve $name given [$outputStr]") + throw RapidsErrorUtils.cannotResolveAttributeError(name, outputStr) } def writePartitionExceedConfigSizeWhenDynamicPartitionError( diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuDataSourceBase.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuDataSourceBase.scala index 0ec720733e8..5589bca0435 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuDataSourceBase.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuDataSourceBase.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,7 +44,7 @@ import org.apache.spark.sql.execution.datasources.v2.orc.OrcDataSourceV2 import org.apache.spark.sql.execution.streaming._ import org.apache.spark.sql.execution.streaming.sources.{RateStreamProvider, TextSocketSourceProvider} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.rapids.shims.SchemaUtilsShims +import org.apache.spark.sql.rapids.shims.{RapidsErrorUtils, SchemaUtilsShims} import org.apache.spark.sql.sources._ import org.apache.spark.sql.types.{DataType, StructType} import org.apache.spark.util.{HadoopFSUtils, ThreadUtils, Utils} @@ -144,8 +144,8 @@ abstract class GpuDataSourceBase( } inferredOpt }.getOrElse { - throw new AnalysisException(s"Failed to resolve the schema for $format for " + - s"the partition column: $partitionColumn. It must be specified manually.") + throw RapidsErrorUtils. + partitionColumnNotSpecifiedError(format.toString, partitionColumn) } } StructType(partitionFields) @@ -162,8 +162,7 @@ abstract class GpuDataSourceBase( caseInsensitiveOptions - "path", SparkShimImpl.filesFromFileIndex(tempFileIndex)) }.getOrElse { - throw new AnalysisException( - s"Unable to infer schema for $format. It must be specified manually.") + throw RapidsErrorUtils.dataSchemaNotSpecifiedError(format.toString) } // We just print a waring message if the data schema and partition schema have the duplicate @@ -201,17 +200,13 @@ abstract class GpuDataSourceBase( case (dataSource: RelationProvider, None) => dataSource.createRelation(sparkSession.sqlContext, caseInsensitiveOptions) case (_: SchemaRelationProvider, None) => - throw new AnalysisException(s"A schema needs to be specified when using $className.") + throw RapidsErrorUtils.schemaNotSpecifiedForSchemaRelationProviderError(className) case (dataSource: RelationProvider, Some(schema)) => val baseRelation = dataSource.createRelation(sparkSession.sqlContext, caseInsensitiveOptions) if (!DataType.equalsIgnoreCompatibleNullability(baseRelation.schema, schema)) { - throw new AnalysisException( - "The user-specified schema doesn't match the actual schema: " + - s"user-specified: ${schema.toDDL}, actual: ${baseRelation.schema.toDDL}. If " + - "you're using DataFrameReader.schema API or creating a table, please do not " + - "specify the schema. Or if you're scanning an existed table, please drop " + - "it and re-create it.") + throw RapidsErrorUtils.userSpecifiedSchemaMismatchActualSchemaError(schema, + baseRelation.schema) } baseRelation @@ -233,9 +228,8 @@ abstract class GpuDataSourceBase( caseInsensitiveOptions - "path", SparkShimImpl.filesFromFileIndex(fileCatalog)) }.getOrElse { - throw new AnalysisException( - s"Unable to infer schema for $format at ${fileCatalog.allFiles().mkString(",")}. " + - "It must be specified manually") + throw RapidsErrorUtils. + dataSchemaNotSpecifiedError(format.toString, fileCatalog.allFiles().mkString(",")) } HadoopFsRelation( @@ -276,8 +270,7 @@ abstract class GpuDataSourceBase( caseInsensitiveOptions)(sparkSession) case _ => - throw new AnalysisException( - s"$className is not a valid Spark SQL Data Source.") + throw RapidsErrorUtils.invalidDataSourceError(className) } relation match { @@ -411,22 +404,13 @@ object GpuDataSourceBase extends Logging { dataSource case Failure(error) => if (provider1.startsWith("org.apache.spark.sql.hive.orc")) { - throw new AnalysisException( - "Hive built-in ORC data source must be used with Hive support enabled. " + - "Please use the native ORC data source by setting 'spark.sql.orc.impl' to " + - "'native'") + throw RapidsErrorUtils.orcNotUsedWithHiveEnabledError() } else if (provider1.toLowerCase(Locale.ROOT) == "avro" || provider1 == "com.databricks.spark.avro" || provider1 == "org.apache.spark.sql.avro") { - throw new AnalysisException( - s"Failed to find data source: $provider1. Avro is built-in but external data " + - "source module since Spark 2.4. Please deploy the application as per " + - "the deployment section of \"Apache Avro Data Source Guide\".") + throw RapidsErrorUtils.failedToFindAvroDataSourceError(provider1) } else if (provider1.toLowerCase(Locale.ROOT) == "kafka") { - throw new AnalysisException( - s"Failed to find data source: $provider1. Please deploy the application as " + - "per the deployment section of " + - "\"Structured Streaming + Kafka Integration Guide\".") + throw RapidsErrorUtils.failedToFindKafkaDataSourceError(provider1) } else { throw new ClassNotFoundException( s"Failed to find data source: $provider1. Please find packages at " + @@ -459,8 +443,7 @@ object GpuDataSourceBase extends Logging { s"defaulting to the internal datasource (${internalSources.head.getClass.getName}).") internalSources.head.getClass } else { - throw new AnalysisException(s"Multiple sources found for $provider1 " + - s"(${sourceNames.mkString(", ")}), please specify the fully qualified class name.") + throw RapidsErrorUtils.findMultipleDataSourceError(provider1, sourceNames) } } } catch { @@ -513,7 +496,7 @@ object GpuDataSourceBase extends Logging { } if (checkEmptyGlobPath && globResult.isEmpty) { - throw new AnalysisException(s"Path does not exist: $globPath") + throw RapidsErrorUtils.dataPathNotExistError(globPath.toString) } globResult @@ -527,7 +510,7 @@ object GpuDataSourceBase extends Logging { ThreadUtils.parmap(nonGlobPaths, "checkPathsExist", numThreads) { path => val fs = path.getFileSystem(hadoopConf) if (!fs.exists(path)) { - throw new AnalysisException(s"Path does not exist: $path") + throw RapidsErrorUtils.dataPathNotExistError(path.toString) } } } catch { diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala index 2b7974fd1a6..ece5ef5acf5 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ import com.nvidia.spark.rapids.{ColumnarFileFormat, GpuDataWritingCommand} import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.internal.io.FileCommitProtocol -import org.apache.spark.sql.{AnalysisException, SaveMode, SparkSession} +import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, CatalogTablePartition} import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.getPartitionPathString @@ -33,7 +33,7 @@ import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.command.{AlterTableAddPartitionCommand, AlterTableDropPartitionCommand, CommandUtils} import org.apache.spark.sql.execution.datasources.{FileFormatWriter, FileIndex, PartitioningUtils} import org.apache.spark.sql.internal.SQLConf.PartitionOverwriteMode -import org.apache.spark.sql.rapids.shims.SchemaUtilsShims +import org.apache.spark.sql.rapids.shims.{RapidsErrorUtils, SchemaUtilsShims} import org.apache.spark.sql.vectorized.ColumnarBatch case class GpuInsertIntoHadoopFsRelationCommand( @@ -121,7 +121,7 @@ case class GpuInsertIntoHadoopFsRelationCommand( val pathExists = fs.exists(qualifiedOutputPath) (mode, pathExists) match { case (SaveMode.ErrorIfExists, true) => - throw new AnalysisException(s"path $qualifiedOutputPath already exists.") + throw RapidsErrorUtils.outputPathAlreadyExistsError(qualifiedOutputPath) case (SaveMode.Overwrite, true) => if (ifPartitionNotExists && matchingPartitions.nonEmpty) { false diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/catalyst/expressions/GpuRandomExpressions.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/catalyst/expressions/GpuRandomExpressions.scala index 6675f678f6d..f9d0be81505 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/catalyst/expressions/GpuRandomExpressions.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/catalyst/expressions/GpuRandomExpressions.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,8 +23,8 @@ import com.nvidia.spark.rapids.Arm.withResource import com.nvidia.spark.rapids.shims.ShimUnaryExpression import org.apache.spark.TaskContext -import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, ExpressionWithRandomSeed} +import org.apache.spark.sql.rapids.execution.RapidsAnalysisException import org.apache.spark.sql.types._ import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.Utils @@ -52,7 +52,7 @@ case class GpuRand(child: Expression) extends ShimUnaryExpression with GpuExpres @transient protected lazy val seed: Long = child match { case GpuLiteral(s, IntegerType) => s.asInstanceOf[Int] case GpuLiteral(s, LongType) => s.asInstanceOf[Long] - case _ => throw new AnalysisException( + case _ => throw new RapidsAnalysisException( s"Input argument to $prettyName must be an integer, long or null literal.") } diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/TrampolineUtil.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/TrampolineUtil.scala index 5ffe08348f1..8a88cc4024d 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/TrampolineUtil.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/TrampolineUtil.scala @@ -157,9 +157,6 @@ object TrampolineUtil { TaskContext.get.taskMemoryManager() } - /** Throw a Spark analysis exception */ - def throwAnalysisException(msg: String) = throw new AnalysisException(msg) - /** Set the task context for the current thread */ def setTaskContext(tc: TaskContext): Unit = TaskContext.setTaskContext(tc) @@ -241,4 +238,13 @@ object TrampolineUtil { } def getSparkHadoopUtilConf: Configuration = SparkHadoopUtil.get.conf + } + +/** + * This class is to only be used to throw errors specific to the + * RAPIDS Accelerator or errors mirroring Spark where a raw + * AnalysisException is thrown directly rather than via an error + * utility class (this should be rare). + */ +class RapidsAnalysisException(msg: String) extends AnalysisException(msg) diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala index fd48b8b6375..4d6d4967a80 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,8 @@ import org.apache.parquet.schema.OriginalType._ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._ import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.rapids.execution.TrampolineUtil +import org.apache.spark.sql.rapids.execution.RapidsAnalysisException +import org.apache.spark.sql.rapids.shims.RapidsErrorUtils import org.apache.spark.sql.types._ object ParquetSchemaClipShims { @@ -64,13 +65,13 @@ object ParquetSchemaClipShims { if (originalType == null) s"$typeName" else s"$typeName ($originalType)" def typeNotSupported() = - TrampolineUtil.throwAnalysisException(s"Parquet type not supported: $typeString") + throw new RapidsAnalysisException(s"Parquet type not supported: $typeString") def typeNotImplemented() = - TrampolineUtil.throwAnalysisException(s"Parquet type not yet supported: $typeString") + throw RapidsErrorUtils.parquetTypeUnsupportedYetError(typeString) def illegalType() = - TrampolineUtil.throwAnalysisException(s"Illegal Parquet type: $typeString") + throw RapidsErrorUtils.illegalParquetTypeError(typeString) // When maxPrecision = -1, we skip precision range check, and always respect the precision // specified in field.getDecimalMetadata. This is useful when interpreting decimal types stored @@ -80,8 +81,7 @@ object ParquetSchemaClipShims { val scale = field.getDecimalMetadata.getScale if (!(maxPrecision == -1 || 1 <= precision && precision <= maxPrecision)) { - TrampolineUtil.throwAnalysisException( - s"Invalid decimal precision: $typeName " + + throw new RapidsAnalysisException(s"Invalid decimal precision: $typeName " + s"cannot store $precision digits (max $maxPrecision)") } @@ -121,7 +121,7 @@ object ParquetSchemaClipShims { case INT96 => if (!SQLConf.get.isParquetINT96AsTimestamp) { - TrampolineUtil.throwAnalysisException( + throw new RapidsAnalysisException( "INT96 is not supported unless it's interpreted as timestamp. " + s"Please try to set ${SQLConf.PARQUET_INT96_AS_TIMESTAMP.key} to true.") } diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala index 6d4ca5da7c3..2ea0301fa2c 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala @@ -45,7 +45,7 @@ import org.apache.hadoop.hive.ql.ErrorMsg import org.apache.hadoop.hive.ql.plan.TableDesc import org.apache.spark.SparkException -import org.apache.spark.sql.{AnalysisException, SparkSession} +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, ExternalCatalog, ExternalCatalogUtils, ExternalCatalogWithListener} import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -58,6 +58,7 @@ import org.apache.spark.sql.hive.client.HiveClientImpl import org.apache.spark.sql.hive.client.hive._ import org.apache.spark.sql.hive.execution.InsertIntoHiveTable import org.apache.spark.sql.hive.rapids.{GpuHiveFileFormat, GpuSaveAsHiveFile, RapidsHiveErrors} +import org.apache.spark.sql.rapids.shims.RapidsErrorUtils import org.apache.spark.sql.vectorized.ColumnarBatch final class GpuInsertIntoHiveTableMeta(cmd: InsertIntoHiveTable, @@ -193,7 +194,7 @@ case class GpuInsertIntoHiveTable( // Report error if any static partition appears after a dynamic partition val isDynamic = partitionColumnNames.map(partitionSpec(_).isEmpty) if (isDynamic.init.zip(isDynamic.tail).contains((true, false))) { - throw new AnalysisException(ErrorMsg.PARTITION_DYN_STA_ORDER.getMsg) + throw RapidsErrorUtils.dynamicPartitionParentError } } diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala index f23229e0956..7fa269db71a 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.types.{DataType, Decimal, DecimalType} -object RapidsErrorUtils { +object RapidsErrorUtils extends RapidsQueryErrorUtils { def invalidArrayIndexError(index: Int, numElements: Int, isElementAtF: Boolean = false): ArrayIndexOutOfBoundsException = { // Follow the Spark string format before 3.3.0 diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsQueryErrorUtils.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsQueryErrorUtils.scala new file mode 100644 index 00000000000..266cb4ef54f --- /dev/null +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsQueryErrorUtils.scala @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "311"} +{"spark": "312"} +{"spark": "313"} +spark-rapids-shim-json-lines ***/ + +package org.apache.spark.sql.rapids.shims + +import org.apache.hadoop.fs.Path +import org.apache.hadoop.hive.ql.ErrorMsg + +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.rapids.execution.RapidsAnalysisException +import org.apache.spark.sql.types.StructType + +trait RapidsQueryErrorUtils { + + def outputPathAlreadyExistsError(qualifiedOutputPath: Path): Throwable = { + new AnalysisException(s"path $qualifiedOutputPath already exists.") + } + + def createTableAsSelectWithNonEmptyDirectoryError(tablePath: String, conf: String): Throwable = { + new AnalysisException(s"CREATE-TABLE-AS-SELECT cannot create table with location to a " + + s"non-empty directory $tablePath. To allow overwriting the existing non-empty directory, " + + s"set '$conf' to true.") + } + + def cannotResolveAttributeError(name: String, outputStr: String): Throwable = { + new AnalysisException(s"Unable to resolve $name given [$outputStr]") + } + + def partitionColumnNotSpecifiedError(format: String, partitionColumn: String): Throwable = { + new AnalysisException(s"Failed to resolve the schema for $format for the partition column: " + + s"$partitionColumn. It must be specified manually.") + } + + def dataSchemaNotSpecifiedError(format: String): Throwable = { + new AnalysisException(s"Unable to infer schema for $format. It must be specified manually.") + } + + def schemaNotSpecifiedForSchemaRelationProviderError(className: String): Throwable = { + new AnalysisException(s"A schema needs to be specified when using $className.") + } + + def userSpecifiedSchemaMismatchActualSchemaError( + schema: StructType, + actualSchema: StructType): Throwable = { + new AnalysisException("The user-specified schema doesn't match the actual schema: " + + s"user-specified: ${schema.toDDL}, actual: ${actualSchema.toDDL}. If " + + "you're using DataFrameReader.schema API or creating a table, please do not " + + "specify the schema. Or if you're scanning an existed table, please drop " + + "it and re-create it.") + } + + def dataSchemaNotSpecifiedError(format: String, fileCatalog: String): Throwable = { + new AnalysisException(s"Unable to infer schema for $format at $fileCatalog. " + + "It must be specified manually") + } + + def invalidDataSourceError(className: String): Throwable = { + new AnalysisException(s"$className is not a valid Spark SQL Data Source.") + } + + def orcNotUsedWithHiveEnabledError(): Throwable = { + new AnalysisException( + s"Hive built-in ORC data source must be used with Hive support enabled. " + + s"Please use the native ORC data source by setting 'spark.sql.orc.impl' to 'native'.") + } + + def failedToFindAvroDataSourceError(provider: String): Throwable = { + new AnalysisException( + s"Failed to find data source: $provider. Avro is built-in but external data " + + "source module since Spark 2.4. Please deploy the application as per " + + "the deployment section of \"Apache Avro Data Source Guide\".") + } + + def failedToFindKafkaDataSourceError(provider: String): Throwable = { + new AnalysisException( + s"Failed to find data source: $provider. Please deploy the application as " + + "per the deployment section of " + + "\"Structured Streaming + Kafka Integration Guide\".") + } + + def findMultipleDataSourceError(provider: String, sourceNames: Seq[String]): Throwable = { + new AnalysisException( + s"Multiple sources found for $provider " + + s"(${sourceNames.mkString(", ")}), please specify the fully qualified class name.") + } + + def dataPathNotExistError(path: String): Throwable = { + new AnalysisException(s"Path does not exist: $path") + } + + def dynamicPartitionParentError: Throwable = { + throw new RapidsAnalysisException(ErrorMsg.PARTITION_DYN_STA_ORDER.getMsg) + } + + def tableOrViewAlreadyExistsError(tableName: String): Throwable = { + new AnalysisException(s"Table $tableName already exists. You need to drop it first.") + } + + def parquetTypeUnsupportedYetError(parquetType: String): Throwable = { + new AnalysisException(s"Parquet type not yet supported: $parquetType.") + } + + def illegalParquetTypeError(parquetType: String): Throwable = { + new AnalysisException(s"Illegal Parquet type: $parquetType.") + } +} \ No newline at end of file diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala index c3152a8a235..bba205f267f 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala @@ -29,7 +29,8 @@ import org.apache.parquet.schema.LogicalTypeAnnotation._ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._ import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.rapids.execution.TrampolineUtil +import org.apache.spark.sql.rapids.execution.RapidsAnalysisException +import org.apache.spark.sql.rapids.shims.RapidsErrorUtils import org.apache.spark.sql.types._ object ParquetSchemaClipShims { @@ -67,10 +68,10 @@ object ParquetSchemaClipShims { if (typeAnnotation == null) s"$typeName" else s"$typeName ($typeAnnotation)" def typeNotImplemented() = - TrampolineUtil.throwAnalysisException(s"Parquet type not yet supported: $typeString") + throw RapidsErrorUtils.parquetTypeUnsupportedYetError(typeString) def illegalType() = - TrampolineUtil.throwAnalysisException(s"Illegal Parquet type: $typeString") + throw RapidsErrorUtils.illegalParquetTypeError(typeString) // When maxPrecision = -1, we skip precision range check, and always respect the precision // specified in field.getDecimalMetadata. This is useful when interpreting decimal types stored @@ -82,7 +83,7 @@ object ParquetSchemaClipShims { val scale = decimalLogicalTypeAnnotation.getScale if (!(maxPrecision == -1 || 1 <= precision && precision <= maxPrecision)) { - TrampolineUtil.throwAnalysisException( + throw new RapidsAnalysisException( s"Invalid decimal precision: $typeName " + s"cannot store $precision digits (max $maxPrecision)") } @@ -143,14 +144,14 @@ object ParquetSchemaClipShims { TimestampType case timestamp: TimestampLogicalTypeAnnotation if timestamp.getUnit == TimeUnit.NANOS && ParquetLegacyNanoAsLongShims.legacyParquetNanosAsLong => - TrampolineUtil.throwAnalysisException( + throw new RapidsAnalysisException( "GPU does not support spark.sql.legacy.parquet.nanosAsLong") case _ => illegalType() } case INT96 => if (!SQLConf.get.isParquetINT96AsTimestamp) { - TrampolineUtil.throwAnalysisException( + throw new RapidsAnalysisException( "INT96 is not supported unless it's interpreted as timestamp. " + s"Please try to set ${SQLConf.PARQUET_INT96_AS_TIMESTAMP.key} to true.") } diff --git a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala index b301397255a..68a6ce30569 100644 --- a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors} import org.apache.spark.sql.types.{DataType, Decimal, DecimalType} -object RapidsErrorUtils { +object RapidsErrorUtils extends RapidsQueryErrorUtils { def invalidArrayIndexError(index: Int, numElements: Int, isElementAtF: Boolean = false): ArrayIndexOutOfBoundsException = { // Follow the Spark string format before 3.3.0 diff --git a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsQueryErrorUtils.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsQueryErrorUtils.scala new file mode 100644 index 00000000000..dbc4145ee54 --- /dev/null +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsQueryErrorUtils.scala @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +{"spark": "400"} +spark-rapids-shim-json-lines ***/ + +package org.apache.spark.sql.rapids.shims + +import org.apache.hadoop.fs.Path +import org.apache.hadoop.hive.ql.ErrorMsg + +import org.apache.spark.sql.errors.QueryCompilationErrors +import org.apache.spark.sql.rapids.execution.RapidsAnalysisException +import org.apache.spark.sql.types.StructType + +trait RapidsQueryErrorUtils { + + def outputPathAlreadyExistsError(qualifiedOutputPath: Path): Throwable = { + QueryCompilationErrors.outputPathAlreadyExistsError(qualifiedOutputPath) + } + + def createTableAsSelectWithNonEmptyDirectoryError(tablePath: String, conf: String): Throwable = { + QueryCompilationErrors.createTableAsSelectWithNonEmptyDirectoryError(tablePath) + } + + def cannotResolveAttributeError(name: String, outputStr: String): Throwable = { + QueryCompilationErrors.cannotResolveAttributeError(name, outputStr) + } + + def partitionColumnNotSpecifiedError(format: String, partitionColumn: String): Throwable = { + QueryCompilationErrors.partitionColumnNotSpecifiedError(format, partitionColumn) + } + + def dataSchemaNotSpecifiedError(format: String): Throwable = { + QueryCompilationErrors.dataSchemaNotSpecifiedError(format) + } + + def schemaNotSpecifiedForSchemaRelationProviderError(className: String): Throwable = { + QueryCompilationErrors.schemaNotSpecifiedForSchemaRelationProviderError(className) + } + + def userSpecifiedSchemaMismatchActualSchemaError( + schema: StructType, + actualSchema: StructType): Throwable = { + QueryCompilationErrors.userSpecifiedSchemaMismatchActualSchemaError(schema, actualSchema) + } + + def dataSchemaNotSpecifiedError(format: String, fileCatalog: String): Throwable = { + QueryCompilationErrors.dataSchemaNotSpecifiedError(format, fileCatalog) + } + + def invalidDataSourceError(className: String): Throwable = { + QueryCompilationErrors.invalidDataSourceError(className) + } + + def orcNotUsedWithHiveEnabledError(): Throwable = { + QueryCompilationErrors.orcNotUsedWithHiveEnabledError() + } + + def failedToFindAvroDataSourceError(provider: String): Throwable = { + QueryCompilationErrors.failedToFindAvroDataSourceError(provider) + } + + def failedToFindKafkaDataSourceError(provider: String): Throwable = { + QueryCompilationErrors.failedToFindKafkaDataSourceError(provider) + } + + def findMultipleDataSourceError(provider: String, sourceNames: Seq[String]): Throwable = { + QueryCompilationErrors.findMultipleDataSourceError(provider, sourceNames) + } + + def dataPathNotExistError(path: String): Throwable = { + QueryCompilationErrors.dataPathNotExistError(path) + } + + def tableOrViewAlreadyExistsError(tableName: String): Throwable = { + QueryCompilationErrors.tableOrViewAlreadyExistsError(tableName) + } + + def parquetTypeUnsupportedYetError(parquetType: String): Throwable = { + QueryCompilationErrors.parquetTypeUnsupportedYetError(parquetType) + } + + def illegalParquetTypeError(parquetType: String): Throwable = { + QueryCompilationErrors.illegalParquetTypeError(parquetType) + } + + def dynamicPartitionParentError: Throwable = { + throw new RapidsAnalysisException(ErrorMsg.PARTITION_DYN_STA_ORDER.getMsg) + } +} \ No newline at end of file diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala index 56708017a23..8c395274e07 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala @@ -44,7 +44,8 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._ import org.apache.spark.sql.execution.datasources.parquet.ParquetReadSupport.containsFieldIds import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.rapids.execution.TrampolineUtil +import org.apache.spark.sql.rapids.execution.RapidsAnalysisException +import org.apache.spark.sql.rapids.shims.RapidsErrorUtils import org.apache.spark.sql.types._ object ParquetSchemaClipShims { @@ -109,10 +110,11 @@ object ParquetSchemaClipShims { if (typeAnnotation == null) s"$typeName" else s"$typeName ($typeAnnotation)" def typeNotImplemented() = - TrampolineUtil.throwAnalysisException(s"Parquet type not yet supported: $typeString") + throw RapidsErrorUtils.parquetTypeUnsupportedYetError(typeString) def illegalType() = - TrampolineUtil.throwAnalysisException(s"Illegal Parquet type: $parquetType") + throw RapidsErrorUtils.illegalParquetTypeError(typeString) + // When maxPrecision = -1, we skip precision range check, and always respect the precision // specified in field.getDecimalMetadata. This is useful when interpreting decimal types stored @@ -124,7 +126,7 @@ object ParquetSchemaClipShims { val scale = decimalLogicalTypeAnnotation.getScale if (!(maxPrecision == -1 || 1 <= precision && precision <= maxPrecision)) { - TrampolineUtil.throwAnalysisException(s"Invalid decimal precision: $typeName " + + throw new RapidsAnalysisException(s"Invalid decimal precision: $typeName " + s"cannot store $precision digits (max $maxPrecision)") } @@ -183,14 +185,14 @@ object ParquetSchemaClipShims { ParquetTimestampAnnotationShims.timestampTypeForMillisOrMicros(timestamp) case timestamp: TimestampLogicalTypeAnnotation if timestamp.getUnit == TimeUnit.NANOS && ParquetLegacyNanoAsLongShims.legacyParquetNanosAsLong => - TrampolineUtil.throwAnalysisException( + throw new RapidsAnalysisException( "GPU does not support spark.sql.legacy.parquet.nanosAsLong") case _ => illegalType() } case INT96 => if (!SQLConf.get.isParquetINT96AsTimestamp) { - TrampolineUtil.throwAnalysisException( + throw new RapidsAnalysisException( "INT96 is not supported unless it's interpreted as timestamp. " + s"Please try to set ${SQLConf.PARQUET_INT96_AS_TIMESTAMP.key} to true.") } diff --git a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala index bb28c370749..e5cdcd43568 100644 --- a/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala +++ b/sql-plugin/src/main/spark330/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, Decimal, DecimalType} -object RapidsErrorUtils extends RapidsErrorUtilsFor330plus { +object RapidsErrorUtils extends RapidsErrorUtilsFor330plus with RapidsQueryErrorUtils { def mapKeyNotExistError( key: String, diff --git a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala index 1012b28d8b7..7e58a54c921 100644 --- a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala +++ b/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package org.apache.spark.sql.rapids.shims import org.apache.spark.sql.errors.QueryExecutionErrors -object RapidsErrorUtils extends RapidsErrorUtilsBase { +object RapidsErrorUtils extends RapidsErrorUtilsBase with RapidsQueryErrorUtils { def sqlArrayIndexNotStartAtOneError(): RuntimeException = { QueryExecutionErrors.elementAtByIndexZeroError(context = null) } diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala index c8d76f85e5c..42fd5941025 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala @@ -37,7 +37,7 @@ import org.apache.hadoop.hive.ql.ErrorMsg import org.apache.hadoop.hive.ql.plan.TableDesc import org.apache.spark.SparkException -import org.apache.spark.sql.{AnalysisException, SparkSession} +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, ExternalCatalog, ExternalCatalogUtils, ExternalCatalogWithListener} import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -48,6 +48,7 @@ import org.apache.spark.sql.hive.HiveExternalCatalog import org.apache.spark.sql.hive.client.HiveClientImpl import org.apache.spark.sql.hive.execution.InsertIntoHiveTable import org.apache.spark.sql.hive.rapids.{GpuHiveFileFormat, GpuSaveAsHiveFile, RapidsHiveErrors} +import org.apache.spark.sql.rapids.shims.RapidsErrorUtils import org.apache.spark.sql.vectorized.ColumnarBatch final class GpuInsertIntoHiveTableMeta(cmd: InsertIntoHiveTable, @@ -182,7 +183,7 @@ case class GpuInsertIntoHiveTable( // Report error if any static partition appears after a dynamic partition val isDynamic = partitionColumnNames.map(partitionSpec(_).isEmpty) if (isDynamic.init.zip(isDynamic.tail).contains((true, false))) { - throw new AnalysisException(ErrorMsg.PARTITION_DYN_STA_ORDER.getMsg) + throw RapidsErrorUtils.dynamicPartitionParentError } } diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala index 78daa0bf6f1..e7b3561f5fd 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala @@ -42,7 +42,7 @@ import org.apache.spark.{SparkException, TaskContext} import org.apache.spark.internal.Logging import org.apache.spark.internal.io.{FileCommitProtocol, SparkHadoopWriterUtils} import org.apache.spark.shuffle.FetchFailedException -import org.apache.spark.sql.{AnalysisException, SparkSession} +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.catalog.BucketSpec import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, AttributeSet, Expression, SortOrder} @@ -51,6 +51,7 @@ import org.apache.spark.sql.connector.write.WriterCommitMessage import org.apache.spark.sql.execution.{SparkPlan, SQLExecution} import org.apache.spark.sql.execution.datasources.{GpuWriteFiles, GpuWriteFilesExec, GpuWriteFilesSpec, WriteTaskResult, WriteTaskStats} import org.apache.spark.sql.execution.datasources.FileFormatWriter.OutputSpec +import org.apache.spark.sql.rapids.execution.RapidsAnalysisException import org.apache.spark.sql.types.StructType import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.{SerializableConfiguration, Utils} @@ -61,7 +62,7 @@ object GpuFileFormatWriter extends Logging { private def verifySchema(format: ColumnarFileFormat, schema: StructType): Unit = { schema.foreach { field => if (!format.supportDataType(field.dataType)) { - throw new AnalysisException( + throw new RapidsAnalysisException( s"$format data source does not support ${field.dataType.catalogString} data type.") } } diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala index 9e36cf41fad..6308f24c552 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala @@ -64,7 +64,7 @@ case class GpuCreateDataSourceTableAsSelectCommand( s"Expect the table $tableName has been dropped when the save mode is Overwrite") if (mode == SaveMode.ErrorIfExists) { - throw new AnalysisException(s"Table $tableName already exists. You need to drop it first.") + throw RapidsErrorUtils.tableOrViewAlreadyExistsError(tableName) } if (mode == SaveMode.Ignore) { // Since the table already exists and the save mode is Ignore, we will just return. diff --git a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala index 8ee0485ab36..e6f8886f19c 100644 --- a/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala +++ b/sql-plugin/src/main/spark340/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala @@ -31,7 +31,7 @@ import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, Decimal, DecimalType} -object RapidsErrorUtils extends RapidsErrorUtilsFor330plus { +object RapidsErrorUtils extends RapidsErrorUtilsFor330plus with RapidsQueryErrorUtils { def mapKeyNotExistError( key: String, diff --git a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala index a0ba17f9bd4..9b800d4e51a 100644 --- a/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ package org.apache.spark.sql.rapids.shims import org.apache.spark.sql.errors.QueryExecutionErrors -object RapidsErrorUtils extends RapidsErrorUtilsBase { +object RapidsErrorUtils extends RapidsErrorUtilsBase with RapidsQueryErrorUtils { def sqlArrayIndexNotStartAtOneError(): RuntimeException = { QueryExecutionErrors.invalidIndexOfZeroError(context = null) } From 3111e2ba20c45ecb245e55b141a5dc2f9fe41d7d Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Tue, 4 Jun 2024 14:36:53 -0700 Subject: [PATCH 17/79] Move Support for `RaiseError` to a Shim Excluding Spark 4.0.0 [databricks] (#10970) * Incomplete impl of RaiseError for 400 * Removed RaiseError from 400 * Signing off Signed-off-by: Raza Jafri --------- Signed-off-by: Raza Jafri --- .../nvidia/spark/rapids/GpuOverrides.scala | 11 +--- .../spark/rapids/shims/RaiseErrorShim.scala | 62 +++++++++++++++++++ .../apache/spark/sql/rapids/shims}/misc.scala | 32 +++++++++- .../spark/rapids/shims/RaiseErrorShim.scala | 28 +++++++++ 4 files changed, 121 insertions(+), 12 deletions(-) create mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala rename sql-plugin/src/main/{scala/org/apache/spark/sql/rapids => spark311/scala/org/apache/spark/sql/rapids/shims}/misc.scala (75%) create mode 100644 sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 55b883a479b..295480d24cc 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -3797,14 +3797,6 @@ object GpuOverrides extends Logging { TypeSig.ARRAY.nested(TypeSig.all)), (e, conf, p, r) => new GpuGetArrayStructFieldsMeta(e, conf, p, r) ), - expr[RaiseError]( - "Throw an exception", - ExprChecks.unaryProject( - TypeSig.NULL, TypeSig.NULL, - TypeSig.STRING, TypeSig.STRING), - (a, conf, p, r) => new UnaryExprMeta[RaiseError](a, conf, p, r) { - override def convertToGpu(child: Expression): GpuExpression = GpuRaiseError(child) - }), expr[DynamicPruningExpression]( "Dynamic pruning expression marker", ExprChecks.unaryProject(TypeSig.all, TypeSig.all, TypeSig.BOOLEAN, TypeSig.BOOLEAN), @@ -3820,7 +3812,8 @@ object GpuOverrides extends Logging { val expressions: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = commonExpressions ++ TimeStamp.getExprs ++ GpuHiveOverrides.exprs ++ ZOrderRules.exprs ++ DecimalArithmeticOverrides.exprs ++ - BloomFilterShims.exprs ++ InSubqueryShims.exprs ++ SparkShimImpl.getExprs + BloomFilterShims.exprs ++ InSubqueryShims.exprs ++ RaiseErrorShim.exprs ++ + SparkShimImpl.getExprs def wrapScan[INPUT <: Scan]( scan: INPUT, diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala new file mode 100644 index 00000000000..de433d5f270 --- /dev/null +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/*** spark-rapids-shim-json-lines +{"spark": "311"} +{"spark": "312"} +{"spark": "313"} +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import com.nvidia.spark.rapids.{ExprRule, GpuOverrides} +import com.nvidia.spark.rapids.{ExprChecks, GpuExpression, TypeSig, UnaryExprMeta} + +import org.apache.spark.sql.catalyst.expressions.{Expression, RaiseError} +import org.apache.spark.sql.rapids.shims.GpuRaiseError + +object RaiseErrorShim { + val exprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = { + Seq(GpuOverrides.expr[RaiseError]( + "Throw an exception", + ExprChecks.unaryProject( + TypeSig.NULL, TypeSig.NULL, + TypeSig.STRING, TypeSig.STRING), + (a, conf, p, r) => new UnaryExprMeta[RaiseError](a, conf, p, r) { + override def convertToGpu(child: Expression): GpuExpression = GpuRaiseError(child) + })).map(r => (r.getClassFor.asSubclass(classOf[Expression]), r)).toMap + } +} diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/misc.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/misc.scala similarity index 75% rename from sql-plugin/src/main/scala/org/apache/spark/sql/rapids/misc.scala rename to sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/misc.scala index b32bdfa207c..1ab58ddcbb6 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/misc.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/misc.scala @@ -13,10 +13,36 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +/*** spark-rapids-shim-json-lines +{"spark": "311"} +{"spark": "312"} +{"spark": "313"} +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.rapids.shims -package org.apache.spark.sql.rapids - -import ai.rapids.cudf.{ColumnVector} +import ai.rapids.cudf.ColumnVector import com.nvidia.spark.rapids.{GpuColumnVector, GpuUnaryExpression} import com.nvidia.spark.rapids.Arm.withResource diff --git a/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala new file mode 100644 index 00000000000..70d40fc19a0 --- /dev/null +++ b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/*** spark-rapids-shim-json-lines +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import com.nvidia.spark.rapids.{ExprRule, GpuOverrides} +import com.nvidia.spark.rapids.{ExprChecks, GpuExpression, TypeSig, UnaryExprMeta} + +import org.apache.spark.sql.catalyst.expressions.{Expression, RaiseError} + +object RaiseErrorShim { + val exprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Map.empty +} From 149e0d58e50d815eb741c225f79aa60c2ea805b2 Mon Sep 17 00:00:00 2001 From: Liangcai Li Date: Wed, 5 Jun 2024 09:48:26 +0800 Subject: [PATCH 18/79] Fix a hive write test failure (#10958) This is a bug fix for the hive write tests. In some of the tests on Spak 351, the ProjectExec will fall back to CPU due to missing the GPU version of the MapFromArrays expression. This PR adds the ProjectExec to the allowed list of fallback for Spark 351 and the laters. Signed-off-by: Firestarman --- integration_tests/src/main/python/hive_parquet_write_test.py | 4 ++-- integration_tests/src/main/python/spark_session.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/integration_tests/src/main/python/hive_parquet_write_test.py b/integration_tests/src/main/python/hive_parquet_write_test.py index e74a99f43c7..96976c3a356 100644 --- a/integration_tests/src/main/python/hive_parquet_write_test.py +++ b/integration_tests/src/main/python/hive_parquet_write_test.py @@ -19,7 +19,7 @@ from data_gen import * from hive_write_test import _restricted_timestamp from marks import allow_non_gpu, ignore_order -from spark_session import with_cpu_session, is_before_spark_320 +from spark_session import with_cpu_session, is_before_spark_320, is_spark_351_or_later # Disable the meta conversion from Hive write to FrameData write in Spark, to test # "GpuInsertIntoHiveTable" for Parquet write. @@ -55,7 +55,7 @@ _hive_write_gens = [_hive_basic_gens, _hive_struct_gens, _hive_array_gens, _hive_map_gens] # ProjectExec falls back on databricks due to no GPU version of "MapFromArrays". -fallback_nodes = ['ProjectExec'] if is_databricks_runtime() else [] +fallback_nodes = ['ProjectExec'] if is_databricks_runtime() or is_spark_351_or_later() else [] @allow_non_gpu(*(non_utc_allow + fallback_nodes)) diff --git a/integration_tests/src/main/python/spark_session.py b/integration_tests/src/main/python/spark_session.py index 78e0b08a651..c55f1976497 100644 --- a/integration_tests/src/main/python/spark_session.py +++ b/integration_tests/src/main/python/spark_session.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -220,6 +220,9 @@ def is_spark_341(): def is_spark_350_or_later(): return spark_version() >= "3.5.0" +def is_spark_351_or_later(): + return spark_version() >= "3.5.1" + def is_spark_330(): return spark_version() == "3.3.0" From d51499973e54d2dde61855c24d512cb95ae65223 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Wed, 5 Jun 2024 12:15:51 +0800 Subject: [PATCH 19/79] Speed up the integration tests by running them in parallel on the Databricks cluster (#10953) * Speed up the integration tests by running them in parallel on the Databricks cluster Separate the integration tests into two parts to speed up the testing process. "part2" includes integration tests : "cache_test, delta_lake_test, shuffle_test and pyarrow_test" "part1" includes all other tests except for the ones in "part1" Run each part in parallel on separate Databricks clusters. Signed-off-by: Tim Liu * TEST_MODE variables on Databrick to run integration tests for CI jobs Signed-off-by: Tim Liu --------- Signed-off-by: Tim Liu --- jenkins/Jenkinsfile-blossom.premerge | 33 ++++++++++++++++--- .../Jenkinsfile-blossom.premerge-databricks | 4 ++- jenkins/databricks/test.sh | 23 +++++++------ 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/jenkins/Jenkinsfile-blossom.premerge b/jenkins/Jenkinsfile-blossom.premerge index fb3a5c2f328..d61638d901a 100755 --- a/jenkins/Jenkinsfile-blossom.premerge +++ b/jenkins/Jenkinsfile-blossom.premerge @@ -274,7 +274,7 @@ git --no-pager diff --name-only HEAD \$BASE -- ${PREMERGE_DOCKERFILE} || true""" } } // end of Unit Test stage - stage('Databricks') { + stage('Databricks IT part1') { when { expression { db_build } } @@ -285,17 +285,42 @@ git --no-pager diff --name-only HEAD \$BASE -- ${PREMERGE_DOCKERFILE} || true""" propagate: false, wait: true, parameters: [ string(name: 'REF', value: params.REF), - string(name: 'GITHUB_DATA', value: params.GITHUB_DATA) + string(name: 'GITHUB_DATA', value: params.GITHUB_DATA), + string(name: 'TEST_MODE', value: 'CI_PART1') ]) if ( DBJob.result != 'SUCCESS' ) { // Output Databricks failure logs to uploaded onto the pre-merge PR print(DBJob.getRawBuild().getLog()) // Fail the pipeline - error "Databricks build result : " + DBJob.result + error "Databricks part1 result : " + DBJob.result } } } - } // end of Databricks + } // end of Databricks IT part1 + + stage('Databricks IT part2') { + when { + expression { db_build } + } + steps { + script { + githubHelper.updateCommitStatus("", "Running - includes databricks", GitHubCommitState.PENDING) + def DBJob = build(job: 'rapids-databricks_premerge-github', + propagate: false, wait: true, + parameters: [ + string(name: 'REF', value: params.REF), + string(name: 'GITHUB_DATA', value: params.GITHUB_DATA), + string(name: 'TEST_MODE', value: 'CI_PART2') + ]) + if ( DBJob.result != 'SUCCESS' ) { + // Output Databricks failure logs to uploaded onto the pre-merge PR + print(DBJob.getRawBuild().getLog()) + // Fail the pipeline + error "Databricks part2 result : " + DBJob.result + } + } + } + } // end of Databricks IT part2 stage('Dummy stage: blue ocean log view') { steps { diff --git a/jenkins/Jenkinsfile-blossom.premerge-databricks b/jenkins/Jenkinsfile-blossom.premerge-databricks index cd872fd9153..5b0a2bf1226 100644 --- a/jenkins/Jenkinsfile-blossom.premerge-databricks +++ b/jenkins/Jenkinsfile-blossom.premerge-databricks @@ -51,6 +51,8 @@ pipeline { description: 'Merged commit of specific PR') string(name: 'GITHUB_DATA', defaultValue: '', description: 'Json-formatted github data from upstream blossom-ci') + choice(name: 'TEST_MODE', choices: ['CI_PART1', 'CI_PART2'], + description: 'Separate integration tests into 2 parts, and run each part in parallell') } environment { @@ -178,7 +180,7 @@ void databricksBuild() { container('cpu') { try { withCredentials([file(credentialsId: 'SPARK_DATABRICKS_PRIVKEY', variable: 'DATABRICKS_PRIVKEY')]) { - def TEST_PARAMS = " -w $DATABRICKS_HOST -t $DATABRICKS_TOKEN -c $CLUSTER_ID" + + def TEST_PARAMS = " -w $DATABRICKS_HOST -t $DATABRICKS_TOKEN -c $CLUSTER_ID -e TEST_MODE=$TEST_MODE" + " -p $DATABRICKS_PRIVKEY -l ./jenkins/databricks/test.sh -v $BASE_SPARK_VERSION -d /home/ubuntu/test.sh" if (params.SPARK_CONF) { TEST_PARAMS += " -f ${params.SPARK_CONF}" diff --git a/jenkins/databricks/test.sh b/jenkins/databricks/test.sh index 404dcd97578..f71f69844f7 100755 --- a/jenkins/databricks/test.sh +++ b/jenkins/databricks/test.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -59,6 +59,7 @@ IS_SPARK_321_OR_LATER=0 # - DELTA_LAKE_ONLY: delta_lake tests only # - MULTITHREADED_SHUFFLE: shuffle tests only # - PYARROW_ONLY: pyarrow tests only +# - CI_PART1 or CI_PART2 : part1 or part2 of the tests run in parallel from CI TEST_MODE=${TEST_MODE:-'DEFAULT'} # Classloader config is here to work around classloader issues with @@ -89,32 +90,30 @@ run_pyarrow_tests() { bash integration_tests/run_pyspark_from_build.sh -m pyarrow_test --pyarrow_test --runtime_env="databricks" --test_type=$TEST_TYPE } -## limit parallelism to avoid OOM kill -export TEST_PARALLEL=${TEST_PARALLEL:-4} - -if [[ $TEST_MODE == "DEFAULT" ]]; then +## Separate the integration tests into "CI_PART1" and "CI_PART2", run each part in parallel on separate Databricks clusters to speed up the testing process. +if [[ $TEST_MODE == "DEFAULT" || $TEST_MODE == "CI_PART1" ]]; then bash integration_tests/run_pyspark_from_build.sh --runtime_env="databricks" --test_type=$TEST_TYPE +fi +## Run tests with jars building from the spark-rapids source code +if [[ "$(pwd)" == "$SOURCE_PATH" ]]; then ## Run cache tests - if [[ "$IS_SPARK_321_OR_LATER" -eq "1" ]]; then + if [[ "$IS_SPARK_321_OR_LATER" -eq "1" && ("$TEST_MODE" == "DEFAULT" || $TEST_MODE == "CI_PART2") ]]; then PYSP_TEST_spark_sql_cache_serializer=${PCBS_CONF} \ bash integration_tests/run_pyspark_from_build.sh --runtime_env="databricks" --test_type=$TEST_TYPE -k cache_test fi -fi -## Run tests with jars building from the spark-rapids source code -if [ "$(pwd)" == "$SOURCE_PATH" ]; then - if [[ "$TEST_MODE" == "DEFAULT" || "$TEST_MODE" == "DELTA_LAKE_ONLY" ]]; then + if [[ "$TEST_MODE" == "DEFAULT" || $TEST_MODE == "CI_PART2" || "$TEST_MODE" == "DELTA_LAKE_ONLY" ]]; then ## Run Delta Lake tests SPARK_SUBMIT_FLAGS="$SPARK_CONF $DELTA_LAKE_CONFS" TEST_PARALLEL=1 \ bash integration_tests/run_pyspark_from_build.sh --runtime_env="databricks" -m "delta_lake" --delta_lake --test_type=$TEST_TYPE fi - if [[ "$TEST_MODE" == "DEFAULT" || "$TEST_MODE" == "MULTITHREADED_SHUFFLE" ]]; then + if [[ "$TEST_MODE" == "DEFAULT" || $TEST_MODE == "CI_PART2" || "$TEST_MODE" == "MULTITHREADED_SHUFFLE" ]]; then ## Mutithreaded Shuffle test rapids_shuffle_smoke_test fi - if [[ "$TEST_MODE" == "DEFAULT" || "$TEST_MODE" == "PYARROW_ONLY" ]]; then + if [[ "$TEST_MODE" == "DEFAULT" || $TEST_MODE == "CI_PART2" || "$TEST_MODE" == "PYARROW_ONLY" ]]; then # Pyarrow tests run_pyarrow_tests fi From 4d3b34679bceb9c3b515ffdb024fa9b52033824f Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Wed, 5 Jun 2024 01:09:09 -0700 Subject: [PATCH 20/79] More compilation fixes for Spark 4.0.0 [databricks] (#10978) * More compilation fixes * Signing off Signed-off-by: Raza Jafri * Added shim for UnsafeCudfRow * POM changes Signed-off-by: Raza Jafri * Added shim for UnsafeCudfRow * Revert "POM changes" This reverts commit dc4823b2c87d748dea50a76623d673f5adead8c3. --------- Signed-off-by: Raza Jafri --- .../nvidia/spark/rapids/CudfUnsafeRow.java | 400 ------------------ .../InternalRowToColumnarBatchIterator.java | 1 + .../spark/rapids/GpuColumnarToRowExec.scala | 4 +- .../spark/rapids/GpuRowToColumnarExec.scala | 2 +- .../spark/rapids/shims/CudfUnsafeRow.scala | 51 +++ .../rapids/shims/CudfUnsafeRowBase.scala | 245 +++++++++++ .../rapids/shims/GpuShuffleExchangeExec.scala | 1 + .../spark/rapids/shims/GpuOrcDataReader.scala | 88 +--- .../shims/GpuOrcDataReader320Plus.scala | 130 ++++++ .../spark/rapids/shims/CudfUnsafeRow.scala | 33 ++ .../spark/rapids/shims/GpuBatchScanExec.scala | 2 +- .../spark/rapids/shims/GpuOrcDataReader.scala | 38 ++ 12 files changed, 505 insertions(+), 490 deletions(-) delete mode 100644 sql-plugin/src/main/java/com/nvidia/spark/rapids/CudfUnsafeRow.java create mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala create mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRowBase.scala create mode 100644 sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader320Plus.scala create mode 100644 sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala create mode 100644 sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala diff --git a/sql-plugin/src/main/java/com/nvidia/spark/rapids/CudfUnsafeRow.java b/sql-plugin/src/main/java/com/nvidia/spark/rapids/CudfUnsafeRow.java deleted file mode 100644 index d25500a77b2..00000000000 --- a/sql-plugin/src/main/java/com/nvidia/spark/rapids/CudfUnsafeRow.java +++ /dev/null @@ -1,400 +0,0 @@ -/* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.nvidia.spark.rapids; - -import org.apache.spark.sql.catalyst.InternalRow; -import org.apache.spark.sql.catalyst.expressions.Attribute; -import org.apache.spark.sql.catalyst.expressions.SpecializedGettersReader; -import org.apache.spark.sql.catalyst.util.ArrayData; -import org.apache.spark.sql.catalyst.util.MapData; -import org.apache.spark.sql.types.DataType; -import org.apache.spark.sql.types.Decimal; -import org.apache.spark.unsafe.Platform; -import org.apache.spark.unsafe.array.ByteArrayMethods; -import org.apache.spark.unsafe.hash.Murmur3_x86_32; -import org.apache.spark.unsafe.types.CalendarInterval; -import org.apache.spark.unsafe.types.UTF8String; - -import java.util.Arrays; - -/** - * This is an InternalRow implementation based off of UnsafeRow, but follows a format for use with - * the row format supported by cudf. In this format each column is padded to match the alignment - * needed by it, and validity is placed at the end one byte at a time. - * - * It also supports remapping the columns so that if the columns were re-ordered to reduce packing - * in the format, then they can be mapped back to their original positions. - * - * This class is likely to go away once we move to code generation when going directly to an - * UnsafeRow through code generation. This is rather difficult because of some details in how - * UnsafeRow works. - */ -public final class CudfUnsafeRow extends InternalRow { - public static int alignOffset(int offset, int alignment) { - return (offset + alignment - 1) & -alignment; - } - - public static int calculateBitSetWidthInBytes(int numFields) { - return (numFields + 7)/ 8; - } - - public static int getRowSizeEstimate(Attribute[] attributes) { - // This needs to match what is in cudf and what is in the constructor. - int offset = 0; - for (Attribute attr : attributes) { - int length = GpuColumnVector.getNonNestedRapidsType(attr.dataType()).getSizeInBytes(); - offset = alignOffset(offset, length); - offset += length; - } - int bitSetWidthInBytes = calculateBitSetWidthInBytes(attributes.length); - // Each row is 64-bit aligned - return alignOffset(offset + bitSetWidthInBytes, 8); - } - - ////////////////////////////////////////////////////////////////////////////// - // Private fields and methods - ////////////////////////////////////////////////////////////////////////////// - - /** - * Address of where the row is stored in off heap memory. - */ - private long address; - - /** - * For each column the starting location to read from. The index to the is the position in - * the row bytes, not the user faceing ordinal. - */ - private int[] startOffsets; - - /** - * At what point validity data starts. - */ - private int fixedWidthSizeInBytes; - - /** - * The size of this row's backing data, in bytes. - */ - private int sizeInBytes; - - /** - * A mapping from the user facing ordinal to the index in the underlying row. - */ - private int[] remapping; - - /** - * Get the address where a field is stored. - * @param ordinal the user facing ordinal. - * @return the address of the field. - */ - private long getFieldAddressFromOrdinal(int ordinal) { - assertIndexIsValid(ordinal); - int i = remapping[ordinal]; - return address + startOffsets[i]; - } - - /** - * Verify that index is valid for this row. - * @param index in this case the index can be either the user facing ordinal or the index into the - * row. - */ - private void assertIndexIsValid(int index) { - assert index >= 0 : "index (" + index + ") should >= 0"; - assert index < startOffsets.length : "index (" + index + ") should < " + startOffsets.length; - } - - ////////////////////////////////////////////////////////////////////////////// - // Public methods - ////////////////////////////////////////////////////////////////////////////// - - /** - * Construct a new Row. The resulting row won't be usable until `pointTo()` has been called, - * since the value returned by this constructor is equivalent to a null pointer. - * - * @param attributes the schema of what this will hold. This is the schema of the underlying - * row, so if columns were re-ordered it is the attributes of the reordered - * data. - * @param remapping a mapping from the user requested column to the underlying column in the - * backing row. - */ - public CudfUnsafeRow(Attribute[] attributes, int[] remapping) { - int offset = 0; - startOffsets = new int[attributes.length]; - for (int i = 0; i < attributes.length; i++) { - Attribute attr = attributes[i]; - int length = GpuColumnVector.getNonNestedRapidsType(attr.dataType()).getSizeInBytes(); - assert length > 0 : "Only fixed width types are currently supported."; - offset = alignOffset(offset, length); - startOffsets[i] = offset; - offset += length; - } - fixedWidthSizeInBytes = offset; - this.remapping = remapping; - assert startOffsets.length == remapping.length; - } - - // for serializer - public CudfUnsafeRow() {} - - @Override - public int numFields() { return startOffsets.length; } - - /** - * Update this CudfUnsafeRow to point to different backing data. - * - * @param address the address in host memory for this. We should change this to be a - * MemoryBuffer class or something like that. - * @param sizeInBytes the size of this row's backing data, in bytes - */ - public void pointTo(long address, int sizeInBytes) { - assert startOffsets != null && startOffsets.length > 0 : "startOffsets not properly initialized"; - assert sizeInBytes % 8 == 0 : "sizeInBytes (" + sizeInBytes + ") should be a multiple of 8"; - this.address = address; - this.sizeInBytes = sizeInBytes; - } - - @Override - public void update(int ordinal, Object value) { - throw new UnsupportedOperationException(); - } - - @Override - public Object get(int ordinal, DataType dataType) { - // Don't remap the ordinal because it will be remapped in each of the other backing APIs - return SpecializedGettersReader.read(this, ordinal, dataType, true, true); - } - - @Override - public boolean isNullAt(int ordinal) { - int i = remapping[ordinal]; - assertIndexIsValid(i); - int validByteIndex = i / 8; - int validBitIndex = i % 8; - byte b = Platform.getByte(null, address + fixedWidthSizeInBytes + validByteIndex); - return ((1 << validBitIndex) & b) == 0; - } - - @Override - public void setNullAt(int ordinal) { - int i = remapping[ordinal]; - assertIndexIsValid(i); - int validByteIndex = i / 8; - int validBitIndex = i % 8; - byte b = Platform.getByte(null, address + fixedWidthSizeInBytes + validByteIndex); - b = (byte)((b & ~(1 << validBitIndex)) & 0xFF); - Platform.putByte(null, address + fixedWidthSizeInBytes + validByteIndex, b); - } - - @Override - public boolean getBoolean(int ordinal) { - return Platform.getBoolean(null, getFieldAddressFromOrdinal(ordinal)); - } - - @Override - public byte getByte(int ordinal) { - return Platform.getByte(null, getFieldAddressFromOrdinal(ordinal)); - } - - @Override - public short getShort(int ordinal) { - return Platform.getShort(null, getFieldAddressFromOrdinal(ordinal)); - } - - @Override - public int getInt(int ordinal) { - return Platform.getInt(null, getFieldAddressFromOrdinal(ordinal)); - } - - @Override - public long getLong(int ordinal) { - return Platform.getLong(null, getFieldAddressFromOrdinal(ordinal)); - } - - @Override - public float getFloat(int ordinal) { - return Platform.getFloat(null, getFieldAddressFromOrdinal(ordinal)); - } - - @Override - public double getDouble(int ordinal) { - return Platform.getDouble(null, getFieldAddressFromOrdinal(ordinal)); - } - - @Override - public Decimal getDecimal(int ordinal, int precision, int scale) { - if (isNullAt(ordinal)) { - return null; - } - if (precision <= Decimal.MAX_INT_DIGITS()) { - return Decimal.createUnsafe(getInt(ordinal), precision, scale); - } else if (precision <= Decimal.MAX_LONG_DIGITS()) { - return Decimal.createUnsafe(getLong(ordinal), precision, scale); - } else { - throw new IllegalArgumentException("NOT IMPLEMENTED YET"); -// byte[] bytes = getBinary(ordinal); -// BigInteger bigInteger = new BigInteger(bytes); -// BigDecimal javaDecimal = new BigDecimal(bigInteger, scale); -// return Decimal.apply(javaDecimal, precision, scale); - } - } - - @Override - public UTF8String getUTF8String(int ordinal) { -// if (isNullAt(ordinal)) return null; -// final long offsetAndSize = getLong(ordinal); -// final int offset = (int) (offsetAndSize >> 32); -// final int size = (int) offsetAndSize; -// return UTF8String.fromAddress(null, address + offset, size); - throw new IllegalArgumentException("NOT IMPLEMENTED YET"); - } - - @Override - public byte[] getBinary(int ordinal) { -// if (isNullAt(ordinal)) { -// return null; -// } else { -// final long offsetAndSize = getLong(ordinal); -// final int offset = (int) (offsetAndSize >> 32); -// final int size = (int) offsetAndSize; -// final byte[] bytes = new byte[size]; -// Platform.copyMemory( -// null, -// address + offset, -// bytes, -// Platform.BYTE_ARRAY_OFFSET, -// size -// ); -// return bytes; -// } - throw new IllegalArgumentException("NOT IMPLEMENTED YET"); - } - - @Override - public CalendarInterval getInterval(int ordinal) { -// if (isNullAt(ordinal)) { -// return null; -// } else { -// final long offsetAndSize = getLong(ordinal); -// final int offset = (int) (offsetAndSize >> 32); -// final int months = Platform.getInt(baseObject, address + offset); -// final int days = Platform.getInt(baseObject, address + offset + 4); -// final long microseconds = Platform.getLong(baseObject, address + offset + 8); -// return new CalendarInterval(months, days, microseconds); -// } - throw new IllegalArgumentException("NOT IMPLEMENTED YET"); - } - - @Override - public CudfUnsafeRow getStruct(int ordinal, int numFields) { -// if (isNullAt(ordinal)) { -// return null; -// } else { -// final long offsetAndSize = getLong(ordinal); -// final int offset = (int) (offsetAndSize >> 32); -// final int size = (int) offsetAndSize; -// final UnsafeRow row = new UnsafeRow(numFields); -// row.pointTo(baseObject, address + offset, size); -// return row; -// } - throw new IllegalArgumentException("NOT IMPLEMENTED YET"); - } - - @Override - public ArrayData getArray(int ordinal) { -// if (isNullAt(ordinal)) { -// return null; -// } else { -// final long offsetAndSize = getLong(ordinal); -// final int offset = (int) (offsetAndSize >> 32); -// final int size = (int) offsetAndSize; -// final UnsafeArrayData array = new UnsafeArrayData(); -// array.pointTo(baseObject, address + offset, size); -// return array; -// } - throw new IllegalArgumentException("NOT IMPLEMENTED YET"); - } - - @Override - public MapData getMap(int ordinal) { -// if (isNullAt(ordinal)) { -// return null; -// } else { -// final long offsetAndSize = getLong(ordinal); -// final int offset = (int) (offsetAndSize >> 32); -// final int size = (int) offsetAndSize; -// final UnsafeMapData map = new UnsafeMapData(); -// map.pointTo(baseObject, address + offset, size); -// return map; -// } - throw new IllegalArgumentException("NOT IMPLEMENTED YET"); - } - - /** - * Copies this row, returning a self-contained UnsafeRow that stores its data in an internal - * byte array rather than referencing data stored in a data page. - */ - @Override - public CudfUnsafeRow copy() { -// UnsafeRow rowCopy = new UnsafeRow(numFields); -// final byte[] rowDataCopy = new byte[sizeInBytes]; -// Platform.copyMemory( -// baseObject, -// address, -// rowDataCopy, -// Platform.BYTE_ARRAY_OFFSET, -// sizeInBytes -// ); -// rowCopy.pointTo(rowDataCopy, Platform.BYTE_ARRAY_OFFSET, sizeInBytes); -// return rowCopy; - throw new IllegalArgumentException("NOT IMPLEMENTED YET"); - } - - @Override - public int hashCode() { - return Murmur3_x86_32.hashUnsafeWords(null, address, sizeInBytes, 42); - } - - @Override - public boolean equals(Object other) { - if (other instanceof CudfUnsafeRow) { - CudfUnsafeRow o = (CudfUnsafeRow) other; - return (sizeInBytes == o.sizeInBytes) && - ByteArrayMethods.arrayEquals(null, address, null, o.address, sizeInBytes) && - Arrays.equals(remapping, o.remapping); - } - return false; - } - - // This is for debugging - @Override - public String toString() { - StringBuilder build = new StringBuilder("["); - for (int i = 0; i < sizeInBytes; i += 8) { - if (i != 0) build.append(','); - build.append(java.lang.Long.toHexString(Platform.getLong(null, address + i))); - } - build.append(']'); - build.append(" remapped with "); - build.append(Arrays.toString(remapping)); - return build.toString(); - } - - @Override - public boolean anyNull() { - throw new IllegalArgumentException("NOT IMPLEMENTED YET"); -// return BitSetMethods.anySet(baseObject, address, bitSetWidthInBytes / 8); - } -} \ No newline at end of file diff --git a/sql-plugin/src/main/java/com/nvidia/spark/rapids/InternalRowToColumnarBatchIterator.java b/sql-plugin/src/main/java/com/nvidia/spark/rapids/InternalRowToColumnarBatchIterator.java index 9e532ba394a..0aa3f0978e9 100644 --- a/sql-plugin/src/main/java/com/nvidia/spark/rapids/InternalRowToColumnarBatchIterator.java +++ b/sql-plugin/src/main/java/com/nvidia/spark/rapids/InternalRowToColumnarBatchIterator.java @@ -35,6 +35,7 @@ import ai.rapids.cudf.NvtxRange; import ai.rapids.cudf.Table; import com.nvidia.spark.rapids.jni.RowConversion; +import com.nvidia.spark.rapids.shims.CudfUnsafeRow; import org.apache.spark.TaskContext; import org.apache.spark.sql.catalyst.InternalRow; diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuColumnarToRowExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuColumnarToRowExec.scala index 38b235a36f6..694d6dabbd6 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuColumnarToRowExec.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuColumnarToRowExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import com.nvidia.spark.rapids.RapidsPluginImplicits._ import com.nvidia.spark.rapids.RmmRapidsRetryIterator.{splitSpillableInHalfByRows, withRetryNoSplit} import com.nvidia.spark.rapids.ScalableTaskCompletion.onTaskCompletion import com.nvidia.spark.rapids.jni.RowConversion -import com.nvidia.spark.rapids.shims.ShimUnaryExecNode +import com.nvidia.spark.rapids.shims.{CudfUnsafeRow, ShimUnaryExecNode} import org.apache.spark.TaskContext import org.apache.spark.rdd.RDD diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuRowToColumnarExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuRowToColumnarExec.scala index 99f17cf341a..51b6645d7b7 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuRowToColumnarExec.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuRowToColumnarExec.scala @@ -19,7 +19,7 @@ package com.nvidia.spark.rapids import ai.rapids.cudf.{NvtxColor, NvtxRange} import com.nvidia.spark.rapids.Arm.withResource import com.nvidia.spark.rapids.GpuColumnVector.GpuColumnarBatchBuilder -import com.nvidia.spark.rapids.shims.{GpuTypeShims, ShimUnaryExecNode} +import com.nvidia.spark.rapids.shims.{CudfUnsafeRow, GpuTypeShims, ShimUnaryExecNode} import org.apache.spark.TaskContext import org.apache.spark.broadcast.Broadcast diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala new file mode 100644 index 00000000000..c04d3b2db29 --- /dev/null +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2020-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/*** spark-rapids-shim-json-lines +{"spark": "311"} +{"spark": "312"} +{"spark": "313"} +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import org.apache.spark.sql.catalyst.expressions.Attribute + +final class CudfUnsafeRow( + attributes: Array[Attribute], + remapping: Array[Int]) extends CudfUnsafeRowBase(attributes, remapping) + +object CudfUnsafeRow extends CudfUnsafeRowTrait \ No newline at end of file diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRowBase.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRowBase.scala new file mode 100644 index 00000000000..e5e0bbd3dc6 --- /dev/null +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRowBase.scala @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2020-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/*** spark-rapids-shim-json-lines +{"spark": "311"} +{"spark": "312"} +{"spark": "313"} +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import java.util.Arrays + +import com.nvidia.spark.rapids.GpuColumnVector + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.expressions.SpecializedGettersReader +import org.apache.spark.sql.catalyst.util.ArrayData +import org.apache.spark.sql.catalyst.util.MapData +import org.apache.spark.sql.types.DataType +import org.apache.spark.sql.types.Decimal +import org.apache.spark.unsafe.Platform +import org.apache.spark.unsafe.array.ByteArrayMethods +import org.apache.spark.unsafe.hash.Murmur3_x86_32 +import org.apache.spark.unsafe.types.CalendarInterval +import org.apache.spark.unsafe.types.UTF8String + +abstract class CudfUnsafeRowBase( + protected val attributes: Array[Attribute], + protected val remapping: Array[Int]) extends InternalRow { + protected var address: Long = _ + private var startOffsets: Array[Int] = _ + private var fixedWidthSizeInBytes: Int = _ + protected var sizeInBytes: Int = _ + + def this() = this(null, null) + + init(attributes, remapping) + + private def init(attributes: Array[Attribute], remapping: Array[Int]): Unit = { + var offset = 0 + startOffsets = new Array[Int](attributes.length) + for (i <- attributes.indices) { + val attr = attributes(i) + val length = GpuColumnVector.getNonNestedRapidsType(attr.dataType).getSizeInBytes + assert(length > 0, "Only fixed width types are currently supported.") + offset = CudfUnsafeRow.alignOffset(offset, length) + startOffsets(i) = offset + offset += length + } + fixedWidthSizeInBytes = offset + assert(startOffsets.length == remapping.length) + } + + override def numFields: Int = startOffsets.length + + def pointTo(address: Long, sizeInBytes: Int): Unit = { + assert(startOffsets != null && startOffsets.length > 0, "startOffsets not properly initialized") + assert(sizeInBytes % 8 == 0, s"sizeInBytes ($sizeInBytes) should be a multiple of 8") + this.address = address + this.sizeInBytes = sizeInBytes + } + + override def update(ordinal: Int, value: Any): Unit = throw new UnsupportedOperationException() + + override def get(ordinal: Int, dataType: DataType): Object = { + SpecializedGettersReader.read(this, ordinal, dataType, true, true) + } + + override def isNullAt(ordinal: Int): Boolean = { + val i = remapping(ordinal) + assertIndexIsValid(i) + val validByteIndex = i / 8 + val validBitIndex = i % 8 + val b = Platform.getByte(null, address + fixedWidthSizeInBytes + validByteIndex) + ((1 << validBitIndex) & b) == 0 + } + + override def setNullAt(ordinal: Int): Unit = { + val i = remapping(ordinal) + assertIndexIsValid(i) + val validByteIndex = i / 8 + val validBitIndex = i % 8 + var b = Platform.getByte(null, address + fixedWidthSizeInBytes + validByteIndex) + b = (b & ~(1 << validBitIndex)).toByte + Platform.putByte(null, address + fixedWidthSizeInBytes + validByteIndex, b) + } + + override def getBoolean(ordinal: Int): Boolean = { + Platform.getBoolean(null, getFieldAddressFromOrdinal(ordinal)) + } + + override def getByte(ordinal: Int): Byte = { + Platform.getByte(null, getFieldAddressFromOrdinal(ordinal)) + } + + override def getShort(ordinal: Int): Short = { + Platform.getShort(null, getFieldAddressFromOrdinal(ordinal)) + } + + override def getInt(ordinal: Int): Int = { + Platform.getInt(null, getFieldAddressFromOrdinal(ordinal)) + } + + override def getLong(ordinal: Int): Long = { + Platform.getLong(null, getFieldAddressFromOrdinal(ordinal)) + } + + override def getFloat(ordinal: Int): Float = { + Platform.getFloat(null, getFieldAddressFromOrdinal(ordinal)) + } + + override def getDouble(ordinal: Int): Double = { + Platform.getDouble(null, getFieldAddressFromOrdinal(ordinal)) + } + + override def getDecimal(ordinal: Int, precision: Int, scale: Int): Decimal = { + if (isNullAt(ordinal)) { + null + } else if (precision <= Decimal.MAX_INT_DIGITS) { + Decimal.createUnsafe(getInt(ordinal), precision, scale) + } else if (precision <= Decimal.MAX_LONG_DIGITS) { + Decimal.createUnsafe(getLong(ordinal), precision, scale) + } else { + throw new IllegalArgumentException("NOT IMPLEMENTED YET") + } + } + + override def getUTF8String(ordinal: Int): UTF8String = { + throw new IllegalArgumentException("NOT IMPLEMENTED YET") + } + + override def getBinary(ordinal: Int): Array[Byte] = { + throw new IllegalArgumentException("NOT IMPLEMENTED YET") + } + + override def getInterval(ordinal: Int): CalendarInterval = { + throw new IllegalArgumentException("NOT IMPLEMENTED YET") + } + + override def getStruct(ordinal: Int, numFields: Int): CudfUnsafeRow = { + throw new IllegalArgumentException("NOT IMPLEMENTED YET") + } + + override def getArray(ordinal: Int): ArrayData = { + throw new IllegalArgumentException("NOT IMPLEMENTED YET") + } + + override def getMap(ordinal: Int): MapData = { + throw new IllegalArgumentException("NOT IMPLEMENTED YET") + } + + override def copy(): CudfUnsafeRow = { + throw new IllegalArgumentException("NOT IMPLEMENTED YET") + } + + override def hashCode(): Int = { + Murmur3_x86_32.hashUnsafeWords(null, address, sizeInBytes, 42) + } + + override def equals(other: Any): Boolean = other match { + case o: CudfUnsafeRow => + sizeInBytes == o.sizeInBytes && + ByteArrayMethods.arrayEquals(null, address, null, o.address, sizeInBytes) && + Arrays.equals(this.remapping, o.remapping) + case _ => false + } + + override def toString: String = { + val build = new StringBuilder("[") + for (i <- 0 until sizeInBytes by 8) { + if (i != 0) build.append(',') + build.append(java.lang.Long.toHexString(Platform.getLong(null, address + i))) + } + build.append(']') + build.append(" remapped with ") + build.append(Arrays.toString(remapping)) + build.toString() + } + + override def anyNull(): Boolean = throw new IllegalArgumentException("NOT IMPLEMENTED YET") + + private def getFieldAddressFromOrdinal(ordinal: Int): Long = { + assertIndexIsValid(ordinal) + val i = remapping(ordinal) + address + startOffsets(i) + } + + private def assertIndexIsValid(index: Int): Unit = { + assert(index >= 0, s"index ($index) should >= 0") + assert(index < startOffsets.length, s"index ($index) should < ${startOffsets.length}") + } +} + +trait CudfUnsafeRowTrait { + def alignOffset(offset: Int, alignment: Int): Int = (offset + alignment - 1) & -alignment + + def calculateBitSetWidthInBytes(numFields: Int): Int = (numFields + 7) / 8 + + def getRowSizeEstimate(attributes: Array[Attribute]): Int = { + var offset = 0 + for (attr <- attributes) { + val length = GpuColumnVector.getNonNestedRapidsType(attr.dataType).getSizeInBytes + offset = alignOffset(offset, length) + offset += length + } + val bitSetWidthInBytes = calculateBitSetWidthInBytes(attributes.length) + alignOffset(offset + bitSetWidthInBytes, 8) + } +} \ No newline at end of file diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala index d94c8e54683..2dcad0d4226 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala @@ -58,6 +58,7 @@ case class GpuShuffleExchangeExec( cpuOutputPartitioning: Partitioning) extends GpuShuffleExchangeExecBaseWithMetrics(gpuOutputPartitioning, child) with ShuffleExchangeLike { + def shuffleId: Int = shuffleDependencyColumnar.shuffleId override def otherCopyArgs: Seq[AnyRef] = cpuOutputPartitioning :: Nil diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala index 8c82074b8f5..aec35945b4e 100644 --- a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala @@ -33,101 +33,17 @@ {"spark": "343"} {"spark": "350"} {"spark": "351"} -{"spark": "400"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims -import java.io.EOFException -import java.nio.ByteBuffer -import java.nio.channels.SeekableByteChannel - -import ai.rapids.cudf.HostMemoryBuffer -import com.nvidia.spark.rapids.Arm.closeOnExcept import com.nvidia.spark.rapids.GpuMetric -import com.nvidia.spark.rapids.filecache.FileCache import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.hive.common.io.DiskRangeList -import org.apache.orc.OrcProto -import org.apache.orc.impl.{BufferChunk, BufferChunkList, DataReaderProperties, InStream, OrcCodecPool} +import org.apache.orc.impl.DataReaderProperties class GpuOrcDataReader( props: DataReaderProperties, conf: Configuration, - metrics: Map[String, GpuMetric]) extends GpuOrcDataReaderBase(props, conf, metrics) { - - private class BufferChunkLoader(useDirect: Boolean) extends BlockLoader { - override def loadRemoteBlocks( - baseOffset: Long, - first: DiskRangeList, - last: DiskRangeList, - data: ByteBuffer): DiskRangeList = { - var current = first - val offset = current.getOffset - while (current ne last.next) { - val buffer = if (current eq last) data else data.duplicate() - buffer.position((current.getOffset - offset).toInt) - buffer.limit((current.getEnd - offset).toInt) - current.asInstanceOf[BufferChunk].setChunk(buffer) - // see if the filecache wants any of this data - val cacheToken = FileCache.get.startDataRangeCache(filePathString, - baseOffset + current.getOffset, current.getLength, conf) - cacheToken.foreach { token => - val hmb = closeOnExcept(HostMemoryBuffer.allocate(current.getLength, false)) { hmb => - hmb.setBytes(0, buffer.array(), - buffer.arrayOffset() + buffer.position(), current.getLength) - hmb - } - token.complete(hmb) - } - current = current.next - } - current - } - - override def loadCachedBlock( - chunk: DiskRangeList, - channel: SeekableByteChannel): DiskRangeList = { - val buffer = if (useDirect) { - ByteBuffer.allocateDirect(chunk.getLength) - } else { - ByteBuffer.allocate(chunk.getLength) - } - while (buffer.remaining() > 0) { - if (channel.read(buffer) < 0) { - throw new EOFException(s"Unexpected EOF while reading cached block for $filePathString") - } - } - buffer.flip() - chunk.asInstanceOf[BufferChunk].setChunk(buffer) - chunk - } - } - - override protected def parseStripeFooter(buf: ByteBuffer, size: Int): OrcProto.StripeFooter = { - OrcProto.StripeFooter.parseFrom( - InStream.createCodedInputStream(InStream.create("footer", - new BufferChunk(buf, 0), 0, size, compression))) - } - - override def getCompressionOptions: InStream.StreamOptions = compression - - override def readFileData(chunks: BufferChunkList, forceDirect: Boolean): BufferChunkList = { - if (chunks != null) { - readDiskRanges(chunks.get, 0, new BufferChunkLoader(forceDirect)) - } - chunks - } - - override def close(): Unit = { - if (compression.getCodec != null) { - if (compression.getCodec != null) { - OrcCodecPool.returnCodec(compression.getCodec.getKind, compression.getCodec) - compression.withCodec(null) - } - } - super.close() - } -} + metrics: Map[String, GpuMetric]) extends GpuOrcDataReader320Plus(props, conf, metrics) object GpuOrcDataReader { // File cache is being used, so we want read ranges that can be cached separately diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader320Plus.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader320Plus.scala new file mode 100644 index 00000000000..e28f7001a2b --- /dev/null +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader320Plus.scala @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/*** spark-rapids-shim-json-lines +{"spark": "320"} +{"spark": "321"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +{"spark": "330"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import java.io.EOFException +import java.nio.ByteBuffer +import java.nio.channels.SeekableByteChannel + +import ai.rapids.cudf.HostMemoryBuffer +import com.nvidia.spark.rapids.Arm.closeOnExcept +import com.nvidia.spark.rapids.GpuMetric +import com.nvidia.spark.rapids.filecache.FileCache +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.hive.common.io.DiskRangeList +import org.apache.orc.OrcProto +import org.apache.orc.impl.{BufferChunk, BufferChunkList, DataReaderProperties, InStream, OrcCodecPool} + +abstract class GpuOrcDataReader320Plus( + props: DataReaderProperties, + conf: Configuration, + metrics: Map[String, GpuMetric]) extends GpuOrcDataReaderBase(props, conf, metrics) { + + private class BufferChunkLoader(useDirect: Boolean) extends BlockLoader { + override def loadRemoteBlocks( + baseOffset: Long, + first: DiskRangeList, + last: DiskRangeList, + data: ByteBuffer): DiskRangeList = { + var current = first + val offset = current.getOffset + while (current ne last.next) { + val buffer = if (current eq last) data else data.duplicate() + buffer.position((current.getOffset - offset).toInt) + buffer.limit((current.getEnd - offset).toInt) + current.asInstanceOf[BufferChunk].setChunk(buffer) + // see if the filecache wants any of this data + val cacheToken = FileCache.get.startDataRangeCache(filePathString, + baseOffset + current.getOffset, current.getLength, conf) + cacheToken.foreach { token => + val hmb = closeOnExcept(HostMemoryBuffer.allocate(current.getLength, false)) { hmb => + hmb.setBytes(0, buffer.array(), + buffer.arrayOffset() + buffer.position(), current.getLength) + hmb + } + token.complete(hmb) + } + current = current.next + } + current + } + + override def loadCachedBlock( + chunk: DiskRangeList, + channel: SeekableByteChannel): DiskRangeList = { + val buffer = if (useDirect) { + ByteBuffer.allocateDirect(chunk.getLength) + } else { + ByteBuffer.allocate(chunk.getLength) + } + while (buffer.remaining() > 0) { + if (channel.read(buffer) < 0) { + throw new EOFException(s"Unexpected EOF while reading cached block for $filePathString") + } + } + buffer.flip() + chunk.asInstanceOf[BufferChunk].setChunk(buffer) + chunk + } + } + + override protected def parseStripeFooter(buf: ByteBuffer, size: Int): OrcProto.StripeFooter = { + OrcProto.StripeFooter.parseFrom( + InStream.createCodedInputStream(InStream.create("footer", + new BufferChunk(buf, 0), 0, size, compression))) + } + + override def getCompressionOptions: InStream.StreamOptions = compression + + override def readFileData(chunks: BufferChunkList, forceDirect: Boolean): BufferChunkList = { + if (chunks != null) { + readDiskRanges(chunks.get, 0, new BufferChunkLoader(forceDirect)) + } + chunks + } + + override def close(): Unit = { + if (compression.getCodec != null) { + if (compression.getCodec != null) { + OrcCodecPool.returnCodec(compression.getCodec.getKind, compression.getCodec) + compression.withCodec(null) + } + } + super.close() + } +} \ No newline at end of file diff --git a/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala new file mode 100644 index 00000000000..623005654fc --- /dev/null +++ b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/*** spark-rapids-shim-json-lines +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.unsafe.types.VariantVal + + +final class CudfUnsafeRow( + attributes: Array[Attribute], + remapping: Array[Int]) extends CudfUnsafeRowBase(attributes, remapping) { + def getVariant(ordinal: Int) = { + throw new UnsupportedOperationException("VariantVal is not supported") + } +} + +object CudfUnsafeRow extends CudfUnsafeRowTrait \ No newline at end of file diff --git a/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala index 3c2b649339b..4fc62d82df3 100644 --- a/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala +++ b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala @@ -44,7 +44,7 @@ case class GpuBatchScanExec( spjParams: StoragePartitionJoinParams = StoragePartitionJoinParams() ) extends GpuBatchScanExecBase(scan, runtimeFilters) { - @transient lazy val batch: Batch = if (scan == null) null else scan.toBatch + @transient override lazy val batch: Batch = if (scan == null) null else scan.toBatch // TODO: unify the equal/hashCode implementation for all data source v2 query plans. override def equals(other: Any): Boolean = other match { case other: GpuBatchScanExec => diff --git a/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala new file mode 100644 index 00000000000..8c6a9c793f2 --- /dev/null +++ b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/*** spark-rapids-shim-json-lines +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import com.nvidia.spark.rapids.GpuMetric +import org.apache.hadoop.conf.Configuration +import org.apache.orc.impl.DataReaderProperties + +class GpuOrcDataReader( + props: DataReaderProperties, + conf: Configuration, + metrics: Map[String, GpuMetric]) extends GpuOrcDataReader320Plus(props, conf, metrics) { + override def releaseAllBuffers(): Unit = { + throw new IllegalStateException("should not be trying to release buffers") + } +} + + +object GpuOrcDataReader { + // File cache is being used, so we want read ranges that can be cached separately + val shouldMergeDiskRanges: Boolean = false +} From c7129f5a006e19e4dc592dfd3810c1ac99f7f263 Mon Sep 17 00:00:00 2001 From: Gary Shen Date: Fri, 7 Jun 2024 11:30:11 +0800 Subject: [PATCH 21/79] Add rapids configs to enable GPU running (#10963) Exclude 2 new failed cases Signed-off-by: Gary Shen --- .../sql/rapids/utils/RapidsTestSettings.scala | 2 ++ .../sql/rapids/utils/RapidsTestsTrait.scala | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala index 8fc86cc6dce..181fb14fb00 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala @@ -34,6 +34,8 @@ class RapidsTestSettings extends BackendTestSettings { .exclude("SPARK-35735: Take into account day-time interval fields in cast", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) .exclude("casting to fixed-precision decimals", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) .exclude("SPARK-32828: cast from a derived user-defined type to a base type", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) + .exclude("cast string to timestamp", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) + .exclude("cast string to date", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) enableSuite[RapidsDataFrameAggregateSuite] .exclude("collect functions", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10772")) .exclude("collect functions structs", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10772")) diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestsTrait.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestsTrait.scala index bcac0b8fe2d..69bd4532c71 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestsTrait.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestsTrait.scala @@ -110,6 +110,30 @@ trait RapidsTestsTrait extends RapidsTestsCommonTrait { // .config("spark.rapids.sql.test.enabled", "true") // .config("spark.rapids.sql.test.allowedNonGpu", // "SerializeFromObjectExec,DeserializeToObjectExec,ExternalRDDScanExec") + .config("spark.rapids.sql.castStringToTimestamp.enabled", "true") + .config("spark.rapids.sql.csv.read.decimal.enabled", "true") + .config("spark.rapids.sql.format.avro.enabled", "true") + .config("spark.rapids.sql.format.avro.read.enabled", "true") + .config("spark.rapids.sql.format.hive.text.write.enabled", "true") + .config("spark.rapids.sql.format.json.enabled", "true") + .config("spark.rapids.sql.format.json.read.enabled", "true") + .config("spark.rapids.sql.incompatibleDateFormats.enabled", "true") + .config("spark.rapids.sql.python.gpu.enabled", "true") + .config("spark.rapids.sql.rowBasedUDF.enabled", "true") + .config("spark.rapids.sql.window.collectList.enabled", "true") + .config("spark.rapids.sql.window.collectSet.enabled", "true") + .config("spark.rapids.sql.window.range.byte.enabled", "true") + .config("spark.rapids.sql.window.range.short.enabled", "true") + .config("spark.rapids.sql.expression.Ascii", "true") + .config("spark.rapids.sql.expression.Conv", "true") + .config("spark.rapids.sql.expression.GetJsonObject", "true") + .config("spark.rapids.sql.expression.JsonToStructs", "true") + .config("spark.rapids.sql.expression.JsonTuple", "true") + .config("spark.rapids.sql.expression.StructsToJson", "true") + .config("spark.rapids.sql.exec.CollectLimitExec", "true") + .config("spark.rapids.sql.exec.FlatMapCoGroupsInPandasExec", "true") + .config("spark.rapids.sql.exec.WindowInPandasExec", "true") + .config("spark.rapids.sql.hasExtendedYearValues", "false") .appName("rapids spark plugin running Vanilla Spark UT") _spark = sparkBuilder From 18c2579f5dfd1ed592f9287d936e3597b45a10b0 Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Sat, 8 Jun 2024 11:48:48 +0800 Subject: [PATCH 22/79] Fix Spark UT issues in RapidsDataFrameAggregateSuite (#10943) * Fix Spark UT issues in RapidsDataFrameAggregateSuite Signed-off-by: Haoyang Li * Added SPARK-24788 back Signed-off-by: Haoyang Li --------- Signed-off-by: Haoyang Li --- .../RapidsDataFrameAggregateSuite.scala | 63 +++++++++++++++++-- .../rapids/utils/BackendTestSettings.scala | 1 + .../sql/rapids/utils/RapidsTestSettings.scala | 10 +-- 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsDataFrameAggregateSuite.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsDataFrameAggregateSuite.scala index 5a394a5b0e8..dba811c073c 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsDataFrameAggregateSuite.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsDataFrameAggregateSuite.scala @@ -19,12 +19,67 @@ spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.suites -import org.apache.spark.sql.DataFrameAggregateSuite +import org.apache.spark.sql.{DataFrameAggregateSuite, Row} +import org.apache.spark.sql.functions._ import org.apache.spark.sql.rapids.utils.RapidsSQLTestsTrait +import org.apache.spark.sql.types._ class RapidsDataFrameAggregateSuite extends DataFrameAggregateSuite with RapidsSQLTestsTrait { - // example to show how to replace the logic of an excluded test case in Vanilla Spark - testRapids("collect functions" ) { // "collect functions" was excluded at RapidsTestSettings - // println("...") + import testImplicits._ + + testRapids("collect functions") { + val df = Seq((1, 2), (2, 2), (3, 4)).toDF("a", "b") + checkAnswer( + df.select(sort_array(collect_list($"a")), sort_array(collect_list($"b"))), + Seq(Row(Seq(1, 2, 3), Seq(2, 2, 4))) + ) + checkAnswer( + df.select(sort_array(collect_set($"a")), sort_array(collect_set($"b"))), + Seq(Row(Seq(1, 2, 3), Seq(2, 4))) + ) + + checkDataset( + df.select(sort_array(collect_set($"a")).as("aSet")).as[Set[Int]], + Set(1, 2, 3)) + checkDataset( + df.select(sort_array(collect_set($"b")).as("bSet")).as[Set[Int]], + Set(2, 4)) + checkDataset( + df.select(sort_array(collect_set($"a")), sort_array(collect_set($"b"))) + .as[(Set[Int], Set[Int])], Seq(Set(1, 2, 3) -> Set(2, 4)): _*) + } + + testRapids("collect functions structs") { + val df = Seq((1, 2, 2), (2, 2, 2), (3, 4, 1)) + .toDF("a", "x", "y") + .select($"a", struct($"x", $"y").as("b")) + checkAnswer( + df.select(sort_array(collect_list($"a")), sort_array(collect_list($"b"))), + Seq(Row(Seq(1, 2, 3), Seq(Row(2, 2), Row(2, 2), Row(4, 1)))) + ) + checkAnswer( + df.select(sort_array(collect_set($"a")), sort_array(collect_set($"b"))), + Seq(Row(Seq(1, 2, 3), Seq(Row(2, 2), Row(4, 1)))) + ) + } + + testRapids("SPARK-17641: collect functions should not collect null values") { + val df = Seq(("1", 2), (null, 2), ("1", 4)).toDF("a", "b") + checkAnswer( + df.select(sort_array(collect_list($"a")), sort_array(collect_list($"b"))), + Seq(Row(Seq("1", "1"), Seq(2, 2, 4))) + ) + checkAnswer( + df.select(sort_array(collect_set($"a")), sort_array(collect_set($"b"))), + Seq(Row(Seq("1"), Seq(2, 4))) + ) + } + + testRapids("collect functions should be able to cast to array type with no null values") { + val df = Seq(1, 2).toDF("a") + checkAnswer(df.select(sort_array(collect_list("a")) cast ArrayType(IntegerType, false)), + Seq(Row(Seq(1, 2)))) + checkAnswer(df.select(sort_array(collect_set("a")) cast ArrayType(FloatType, false)), + Seq(Row(Seq(1.0, 2.0)))) } } diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/BackendTestSettings.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/BackendTestSettings.scala index 83396e977fa..e1aec1ffebc 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/BackendTestSettings.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/BackendTestSettings.scala @@ -83,6 +83,7 @@ abstract class BackendTestSettings { // or a description like "This simply can't work on GPU". // It should never be "unknown" or "need investigation" case class KNOWN_ISSUE(reason: String) extends ExcludeReason + case class ADJUST_UT(reason: String) extends ExcludeReason case class WONT_FIX_ISSUE(reason: String) extends ExcludeReason diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala index 181fb14fb00..ad93c4dd2e9 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala @@ -37,11 +37,11 @@ class RapidsTestSettings extends BackendTestSettings { .exclude("cast string to timestamp", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) .exclude("cast string to date", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) enableSuite[RapidsDataFrameAggregateSuite] - .exclude("collect functions", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10772")) - .exclude("collect functions structs", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10772")) - .exclude("collect functions should be able to cast to array type with no null values", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10772")) - .exclude("SPARK-17641: collect functions should not collect null values", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10772")) - .exclude("SPARK-19471: AggregationIterator does not initialize the generated result projection before using it", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10772")) + .exclude("collect functions", ADJUST_UT("order of elements in the array is non-deterministic in collect")) + .exclude("collect functions structs", ADJUST_UT("order of elements in the array is non-deterministic in collect")) + .exclude("collect functions should be able to cast to array type with no null values", ADJUST_UT("order of elements in the array is non-deterministic in collect")) + .exclude("SPARK-17641: collect functions should not collect null values", ADJUST_UT("order of elements in the array is non-deterministic in collect")) + .exclude("SPARK-19471: AggregationIterator does not initialize the generated result projection before using it", WONT_FIX_ISSUE("Codegen related UT, not applicable for GPU")) .exclude("SPARK-24788: RelationalGroupedDataset.toString with unresolved exprs should not fail", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10801")) enableSuite[RapidsJsonExpressionsSuite] .exclude("from_json - invalid data", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) From 9030b133c5770f791da19f6610b9fde1e85a9d50 Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Fri, 7 Jun 2024 22:44:41 -0700 Subject: [PATCH 23/79] Addressing the Spark change of renaming the named parameter (#10992) Signed-off-by: Raza Jafri --- .../spark/sql/rapids/GpuInSubqueryExecSuite.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/GpuInSubqueryExecSuite.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/GpuInSubqueryExecSuite.scala index 82ce1073e13..a606dba0572 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/GpuInSubqueryExecSuite.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/GpuInSubqueryExecSuite.scala @@ -65,7 +65,7 @@ class GpuInSubqueryExecSuite extends SparkQueryCompareTestSuite { private def buildCpuInSubqueryPlan( spark: SparkSession, - shouldBroadcast: Boolean): SparkPlan = { + shouldBroadcastOrDpp: Boolean): SparkPlan = { val df1ReadExec = readToPhysicalPlan(nullableStringsIntsDf(spark)) val df2ReadExec = readToPhysicalPlan(subqueryTable(spark)) val inSubquery = InSubqueryExec( @@ -73,16 +73,19 @@ class GpuInSubqueryExecSuite extends SparkQueryCompareTestSuite { SubqueryExec("sbe", ProjectExec(Seq(df2ReadExec.output.head), df2ReadExec)), ExprId(7), - shouldBroadcast=shouldBroadcast) + shouldBroadcastOrDpp) FilterExec(DynamicPruningExpression(inSubquery), df1ReadExec) } - for (shouldBroadcast <- Seq(false, true)) { - test(s"InSubqueryExec shouldBroadcast=$shouldBroadcast") { + /** + * The named parameter shouldBroadcast was renamed to isDynamicPruning in Spark 4.0.0+ + */ + for (shouldBroadcastOrDpp <- Seq(false, true)) { + test(s"InSubqueryExec shouldBroadcastOrDpp=$shouldBroadcastOrDpp") { val gpuResults = withGpuSparkSession({ spark => val overrides = new GpuOverrides() val transitionOverrides = new GpuTransitionOverrides() - val cpuPlan = buildCpuInSubqueryPlan(spark, shouldBroadcast) + val cpuPlan = buildCpuInSubqueryPlan(spark, shouldBroadcastOrDpp) val gpuPlan = transitionOverrides(overrides(cpuPlan)) gpuPlan.execute().collect() }) From 586e6f4532cac49ab77e3d3d7c75545ada66fd77 Mon Sep 17 00:00:00 2001 From: Advait Chandorkar <110400437+AdvaitChandorkar07@users.noreply.github.com> Date: Tue, 11 Jun 2024 00:04:59 +0530 Subject: [PATCH 24/79] Increase the console output of buildall upon build failures (#10998) Signed-off-by: AdvaitChandorkar07 --- build/buildall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/buildall b/build/buildall index e8c0610deb7..b3c473be141 100755 --- a/build/buildall +++ b/build/buildall @@ -265,7 +265,7 @@ function build_single_shim() { -Dmaven.scaladoc.skip \ -Dmaven.scalastyle.skip="$SKIP_CHECKS" \ -pl tools -am > "$LOG_FILE" 2>&1 || { - [[ "$LOG_FILE" != "/dev/tty" ]] && echo "$LOG_FILE:" && tail -20 "$LOG_FILE" || true + [[ "$LOG_FILE" != "/dev/tty" ]] && echo "$LOG_FILE:" && tail -500 "$LOG_FILE" || true exit 255 } } From f47c205041eb6efc78b162903fc5c2aaf2be30b5 Mon Sep 17 00:00:00 2001 From: Liangcai Li Date: Tue, 11 Jun 2024 09:38:32 +0800 Subject: [PATCH 25/79] Allow ProjectExec fall fallback to CPU for 350 (#11032) This is a bug fix for the hive write tests. This is simliar as #10958 but it fixes the failures for Spark 350. The tests fail because the ProjectExec will fall back to CPU due to missing the GPU version of the MapFromArrays expression. This PR adds the ProjectExec to the allowed list of fallback for Spark 350 and the laters. Signed-off-by: Firestarman --- integration_tests/src/main/python/hive_parquet_write_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/src/main/python/hive_parquet_write_test.py b/integration_tests/src/main/python/hive_parquet_write_test.py index 96976c3a356..f62439a39af 100644 --- a/integration_tests/src/main/python/hive_parquet_write_test.py +++ b/integration_tests/src/main/python/hive_parquet_write_test.py @@ -19,7 +19,7 @@ from data_gen import * from hive_write_test import _restricted_timestamp from marks import allow_non_gpu, ignore_order -from spark_session import with_cpu_session, is_before_spark_320, is_spark_351_or_later +from spark_session import with_cpu_session, is_before_spark_320, is_spark_350_or_later # Disable the meta conversion from Hive write to FrameData write in Spark, to test # "GpuInsertIntoHiveTable" for Parquet write. @@ -55,7 +55,7 @@ _hive_write_gens = [_hive_basic_gens, _hive_struct_gens, _hive_array_gens, _hive_map_gens] # ProjectExec falls back on databricks due to no GPU version of "MapFromArrays". -fallback_nodes = ['ProjectExec'] if is_databricks_runtime() or is_spark_351_or_later() else [] +fallback_nodes = ['ProjectExec'] if is_databricks_runtime() or is_spark_350_or_later() else [] @allow_non_gpu(*(non_utc_allow + fallback_nodes)) From 1ca4c44408e02df3710bb559794861b3ee861afb Mon Sep 17 00:00:00 2001 From: Peixin Date: Tue, 11 Jun 2024 10:01:32 +0800 Subject: [PATCH 26/79] Update blossom-ci ACL to secure format (#11036) Signed-off-by: Peixin Li --- .github/workflows/blossom-ci.yml | 82 ++++++++++++++++---------------- 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml index 6f597d6baf3..3da5fae084d 100644 --- a/.github/workflows/blossom-ci.yml +++ b/.github/workflows/blossom-ci.yml @@ -33,46 +33,48 @@ jobs: args: ${{ env.args }} # This job only runs for pull request comments - if: contains( '\ - abellina,\ - anfeng,\ - firestarman,\ - GaryShen2008,\ - jlowe,\ - kuhushukla,\ - mythrocks,\ - nartal1,\ - nvdbaranec,\ - NvTimLiu,\ - razajafri,\ - revans2,\ - rwlee,\ - sameerz,\ - tgravescs,\ - wbo4958,\ - wjxiz1992,\ - sperlingxx,\ - hyperbolic2346,\ - gerashegalov,\ - ttnghia,\ - nvliyuan,\ - res-life,\ - HaoYang670,\ - NVnavkumar,\ - amahussein,\ - mattahrens,\ - YanxuanLiu,\ - cindyyuanjiang,\ - thirtiseven,\ - winningsix,\ - viadea,\ - yinqingh,\ - parthosa,\ - liurenjie1024,\ - binmahone,\ - zpuller,\ - pxLi,\ - ', format('{0},', github.actor)) && github.event.comment.body == 'build' + if: | + github.event.comment.body == 'build' && + ( + github.actor == 'abellina' || + github.actor == 'anfeng' || + github.actor == 'firestarman' || + github.actor == 'GaryShen2008' || + github.actor == 'jlowe' || + github.actor == 'kuhushukla' || + github.actor == 'mythrocks' || + github.actor == 'nartal1' || + github.actor == 'nvdbaranec' || + github.actor == 'NvTimLiu' || + github.actor == 'razajafri' || + github.actor == 'revans2' || + github.actor == 'rwlee' || + github.actor == 'sameerz' || + github.actor == 'tgravescs' || + github.actor == 'wbo4958' || + github.actor == 'wjxiz1992' || + github.actor == 'sperlingxx' || + github.actor == 'hyperbolic2346' || + github.actor == 'gerashegalov' || + github.actor == 'ttnghia' || + github.actor == 'nvliyuan' || + github.actor == 'res-life' || + github.actor == 'HaoYang670' || + github.actor == 'NVnavkumar' || + github.actor == 'amahussein' || + github.actor == 'mattahrens' || + github.actor == 'YanxuanLiu' || + github.actor == 'cindyyuanjiang' || + github.actor == 'thirtiseven' || + github.actor == 'winningsix' || + github.actor == 'viadea' || + github.actor == 'yinqingh' || + github.actor == 'parthosa' || + github.actor == 'liurenjie1024' || + github.actor == 'binmahone' || + github.actor == 'zpuller' || + github.actor == 'pxLi' + ) steps: - name: Check if comment is issued by authorized person run: blossom-ci From 4e9b961e700726a4515bf3c44e88cfbce4efff9a Mon Sep 17 00:00:00 2001 From: Feng Jiang <106386742+Feng-Jiang28@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:47:15 +0800 Subject: [PATCH 27/79] Append new authorized user to blossom-ci whitelist [skip ci] (#11040) Signed-off-by: fejiang --- .github/workflows/blossom-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml index 3da5fae084d..4b8071303c1 100644 --- a/.github/workflows/blossom-ci.yml +++ b/.github/workflows/blossom-ci.yml @@ -73,7 +73,8 @@ jobs: github.actor == 'liurenjie1024' || github.actor == 'binmahone' || github.actor == 'zpuller' || - github.actor == 'pxLi' + github.actor == 'pxLi' || + github.actor == 'Feng-Jiang28' ) steps: - name: Check if comment is issued by authorized person From 2cf59346fd66e1d500a16e8107068dc9e20c3585 Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Wed, 12 Jun 2024 08:03:27 +0800 Subject: [PATCH 28/79] Rewrite multiple literal choice regex to multiple contains in rlike (#10977) * rewrite multiple literal choice to multiple contains, wip Signed-off-by: Haoyang Li * fix bug Signed-off-by: Haoyang Li * optimize memory Signed-off-by: Haoyang Li * remove debug log Signed-off-by: Haoyang Li * address comments Signed-off-by: Haoyang Li * Apply suggestions from code review Co-authored-by: Gera Shegalov * support abc|def case Signed-off-by: Haoyang Li * fix 2.13 Signed-off-by: Haoyang Li * fix 2.13 build Signed-off-by: Haoyang Li --------- Signed-off-by: Haoyang Li Co-authored-by: Gera Shegalov --- .../src/main/python/regexp_test.py | 8 +- .../com/nvidia/spark/rapids/RegexParser.scala | 94 +++++++++++-------- .../spark/sql/rapids/stringFunctions.scala | 30 +++++- .../RegularExpressionRewriteSuite.scala | 31 ++++-- 4 files changed, 118 insertions(+), 45 deletions(-) diff --git a/integration_tests/src/main/python/regexp_test.py b/integration_tests/src/main/python/regexp_test.py index 89929eb6762..18a83870d83 100644 --- a/integration_tests/src/main/python/regexp_test.py +++ b/integration_tests/src/main/python/regexp_test.py @@ -454,6 +454,7 @@ def test_rlike_rewrite_optimization(): 'rlike(a, "(.*)(abb)(.*)")', 'rlike(a, "^(abb)(.*)")', 'rlike(a, "^abb")', + 'rlike(a, "^.*(aaa)")', 'rlike(a, "\\\\A(abb)(.*)")', 'rlike(a, "\\\\Aabb")', 'rlike(a, "^(abb)\\\\Z")', @@ -466,7 +467,12 @@ def test_rlike_rewrite_optimization(): 'rlike(a, "ab[a-c]{3}")', 'rlike(a, "a[a-c]{1,3}")', 'rlike(a, "a[a-c]{1,}")', - 'rlike(a, "a[a-c]+")'), + 'rlike(a, "a[a-c]+")', + 'rlike(a, "(aaa|bbb|ccc)")', + 'rlike(a, ".*.*(aaa|bbb).*.*")', + 'rlike(a, "^.*(aaa|bbb|ccc)")', + 'rlike(a, "aaa|bbb")', + 'rlike(a, "aaa|(bbb|ccc)")'), conf=_regexp_conf) def test_regexp_replace_character_set_negated(): diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala index 0f5ada9f7fa..1ca155f8a52 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala @@ -2026,6 +2026,7 @@ object RegexOptimizationType { case class Contains(literal: String) extends RegexOptimizationType case class PrefixRange(literal: String, length: Int, rangeStart: Int, rangeEnd: Int) extends RegexOptimizationType + case class MultipleContains(literals: Seq[String]) extends RegexOptimizationType case object NoOptimization extends RegexOptimizationType } @@ -2091,6 +2092,20 @@ object RegexRewrite { } } + private def getMultipleContainsLiterals(ast: RegexAST): Seq[String] = { + ast match { + case RegexGroup(_, term, _) => getMultipleContainsLiterals(term) + case RegexChoice(RegexSequence(parts), ls) if isLiteralString(parts) => { + getMultipleContainsLiterals(ls) match { + case Seq() => Seq.empty + case literals => RegexCharsToString(parts) +: literals + } + } + case RegexSequence(parts) if (isLiteralString(parts)) => Seq(RegexCharsToString(parts)) + case _ => Seq.empty + } + } + private def isWildcard(ast: RegexAST): Boolean = { ast match { case RegexRepetition(RegexChar('.'), SimpleQuantifier('*')) => true @@ -2101,11 +2116,8 @@ object RegexRewrite { } private def stripLeadingWildcards(astLs: collection.Seq[RegexAST]): - collection.Seq[RegexAST] = astLs match { - case (RegexChar('^') | RegexEscaped('A')) :: tail => - // if the pattern starts with ^ or \A, strip it too - tail.dropWhile(isWildcard) - case _ => astLs.dropWhile(isWildcard) + collection.Seq[RegexAST] = { + astLs.dropWhile(isWildcard) } private def stripTailingWildcards(astLs: collection.Seq[RegexAST]): @@ -2124,40 +2136,48 @@ object RegexRewrite { * Matches the given regex ast to a regex optimization type for regex rewrite * optimization. * - * @param ast unparsed children of the Abstract Syntax Tree parsed from a regex pattern. + * @param ast Abstract Syntax Tree parsed from a regex pattern. * @return The `RegexOptimizationType` for the given pattern. */ - @scala.annotation.tailrec - def matchSimplePattern(ast: Seq[RegexAST]): RegexOptimizationType = { - ast match { - case (RegexChar('^') | RegexEscaped('A')) :: astTail => - val noTrailingWildCards = stripTailingWildcards(astTail) - if (isLiteralString(noTrailingWildCards)) { - // ^literal.* => startsWith literal - RegexOptimizationType.StartsWith(RegexCharsToString(noTrailingWildCards)) - } else { - val noWildCards = stripLeadingWildcards(noTrailingWildCards) - if (noWildCards.length == noTrailingWildCards.length) { - // TODO startsWith with PrefIxRange - RegexOptimizationType.NoOptimization - } else { - matchSimplePattern(astTail) - } - } - case astLs => { - val noStartsWithAst = stripTailingWildcards(stripLeadingWildcards(astLs)) - val prefixRangeInfo = getPrefixRangePattern(noStartsWithAst) - if (prefixRangeInfo.isDefined) { - val (prefix, length, start, end) = prefixRangeInfo.get - // (literal[a-b]{x,y}) => prefix range pattern - RegexOptimizationType.PrefixRange(prefix, length, start, end) - } else if (isLiteralString(noStartsWithAst)) { - // literal.* or (literal).* => contains literal - RegexOptimizationType.Contains(RegexCharsToString(noStartsWithAst)) - } else { - RegexOptimizationType.NoOptimization - } + def matchSimplePattern(ast: RegexAST): RegexOptimizationType = { + val astLs = ast match { + case RegexSequence(_) => ast.children() + case _ => Seq(ast) + } + val noTailingWildcards = stripTailingWildcards(astLs) + if (noTailingWildcards.headOption.exists( + ast => ast == RegexChar('^') || ast == RegexEscaped('A'))) { + val possibleLiteral = noTailingWildcards.drop(1) + if (isLiteralString(possibleLiteral)) { + return RegexOptimizationType.StartsWith(RegexCharsToString(possibleLiteral)) + } + } + + val noStartsWithAst = stripLeadingWildcards(noTailingWildcards) + + // Check if the pattern is a contains literal pattern + if (isLiteralString(noStartsWithAst)) { + // literal or .*(literal).* => contains literal + return RegexOptimizationType.Contains(RegexCharsToString(noStartsWithAst)) + } + + // Check if the pattern is a multiple contains literal pattern (e.g. "abc|def|ghi") + if (noStartsWithAst.length == 1) { + val containsLiterals = getMultipleContainsLiterals(noStartsWithAst.head) + if (!containsLiterals.isEmpty) { + return RegexOptimizationType.MultipleContains(containsLiterals) } } + + // Check if the pattern is a prefix range pattern (e.g. "abc[a-z]{3}") + val prefixRangeInfo = getPrefixRangePattern(noStartsWithAst) + if (prefixRangeInfo.isDefined) { + val (prefix, length, start, end) = prefixRangeInfo.get + // (literal[a-b]{x,y}) => prefix range pattern + return RegexOptimizationType.PrefixRange(prefix, length, start, end) + } + + // return NoOptimization if the pattern is not a simple pattern and use cuDF + RegexOptimizationType.NoOptimization } -} \ No newline at end of file +} diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala index 8fea4014149..dc2845e4461 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala @@ -1073,7 +1073,7 @@ class GpuRLikeMeta( val originalPattern = str.toString val regexAst = new RegexParser(originalPattern).parse() if (conf.isRlikeRegexRewriteEnabled) { - rewriteOptimizationType = RegexRewrite.matchSimplePattern(regexAst.children()) + rewriteOptimizationType = RegexRewrite.matchSimplePattern(regexAst) } val (transpiledAST, _) = new CudfRegexTranspiler(RegexFindMode) .getTranspiledAST(regexAst, None, None) @@ -1097,6 +1097,7 @@ class GpuRLikeMeta( } case StartsWith(s) => GpuStartsWith(lhs, GpuLiteral(s, StringType)) case Contains(s) => GpuContains(lhs, GpuLiteral(s, StringType)) + case MultipleContains(ls) => GpuMultipleContains(lhs, ls) case PrefixRange(s, length, start, end) => GpuLiteralRangePattern(lhs, GpuLiteral(s, StringType), length, start, end) case _ => throw new IllegalStateException("Unexpected optimization type") @@ -1126,6 +1127,33 @@ case class GpuRLike(left: Expression, right: Expression, pattern: String) override def dataType: DataType = BooleanType } +case class GpuMultipleContains(input: Expression, searchList: Seq[String]) + extends GpuUnaryExpression with ImplicitCastInputTypes with NullIntolerant { + + override def dataType: DataType = BooleanType + + override def child: Expression = input + + override def inputTypes: Seq[AbstractDataType] = Seq(StringType) + + override def doColumnar(input: GpuColumnVector): ColumnVector = { + assert(searchList.length > 1) + val accInit = withResource(Scalar.fromString(searchList.head)) { searchScalar => + input.getBase.stringContains(searchScalar) + } + searchList.tail.foldLeft(accInit) { (acc, search) => + val containsSearch = withResource(Scalar.fromString(search)) { searchScalar => + input.getBase.stringContains(searchScalar) + } + withResource(acc) { _ => + withResource(containsSearch) { _ => + acc.or(containsSearch) + } + } + } + } +} + case class GpuLiteralRangePattern(left: Expression, right: Expression, length: Int, start: Int, end: Int) extends GpuBinaryExpressionArgsAnyScalar with ImplicitCastInputTypes with NullIntolerant { diff --git a/tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionRewriteSuite.scala b/tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionRewriteSuite.scala index a140f4123f4..7626c1450c1 100644 --- a/tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionRewriteSuite.scala +++ b/tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionRewriteSuite.scala @@ -23,7 +23,7 @@ class RegularExpressionRewriteSuite extends AnyFunSuite { Unit = { val results = patterns.map { pattern => val ast = new RegexParser(pattern).parse() - RegexRewrite.matchSimplePattern(ast.children()) + RegexRewrite.matchSimplePattern(ast) } assert(results == excepted) } @@ -40,9 +40,9 @@ class RegularExpressionRewriteSuite extends AnyFunSuite { test("regex rewrite contains") { import RegexOptimizationType._ val patterns = Seq(".*abc.*", ".*(abc).*", "^.*(abc).*$", "^.*(.*)(abc).*.*", - raw".*\w.*\Z", raw".*..*\Z") - val excepted = Seq(Contains("abc"), Contains("abc"), NoOptimization, Contains("abc"), - NoOptimization, NoOptimization) + raw".*\w.*\Z", raw".*..*\Z", "^(.*)(abc)") + val excepted = Seq(Contains("abc"), Contains("abc"), NoOptimization, NoOptimization, + NoOptimization, NoOptimization, NoOptimization) verifyRewritePattern(patterns, excepted) } @@ -67,8 +67,27 @@ class RegularExpressionRewriteSuite extends AnyFunSuite { PrefixRange("火花急流", 1, 19968, 40869), NoOptimization, // starts with PrefixRange not supported NoOptimization, // starts with PrefixRange not supported - PrefixRange("", 6, 48, 57), - PrefixRange("", 3, 48, 57) + NoOptimization, // .* can't match line break so can't be optimized + NoOptimization // .* can't match line break so can't be optimized + ) + verifyRewritePattern(patterns, excepted) + } + + test("regex rewrite multiple contains") { + import RegexOptimizationType._ + val patterns = Seq( + "(abc|def).*", + ".*(abc|def|ghi).*", + "((abc)|(def))", + "(abc)|(def)", + "(火花|急流)" + ) + val excepted = Seq( + MultipleContains(Seq("abc", "def")), + MultipleContains(Seq("abc", "def", "ghi")), + MultipleContains(Seq("abc", "def")), + MultipleContains(Seq("abc", "def")), + MultipleContains(Seq("火花", "急流")) ) verifyRewritePattern(patterns, excepted) } From d9686d4dba4afd6e2c061fd2822d40d9a55d88d4 Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Wed, 12 Jun 2024 16:26:28 -0500 Subject: [PATCH 29/79] Add in the ability to fingerprint JSON columns (#11002) Signed-off-by: Robert (Bobby) Evans --- .../spark/sql/tests/datagen/bigDataGen.scala | 1181 ++++++++++++++--- 1 file changed, 1002 insertions(+), 179 deletions(-) diff --git a/datagen/src/main/scala/org/apache/spark/sql/tests/datagen/bigDataGen.scala b/datagen/src/main/scala/org/apache/spark/sql/tests/datagen/bigDataGen.scala index 91335afe4e6..14e0d4e0970 100644 --- a/datagen/src/main/scala/org/apache/spark/sql/tests/datagen/bigDataGen.scala +++ b/datagen/src/main/scala/org/apache/spark/sql/tests/datagen/bigDataGen.scala @@ -16,21 +16,22 @@ package org.apache.spark.sql.tests.datagen +import com.fasterxml.jackson.core.{JsonFactoryBuilder, JsonParser, JsonToken} +import com.fasterxml.jackson.core.json.JsonReadFeature import java.math.{BigDecimal => JavaBigDecimal} import java.sql.{Date, Timestamp} import java.time.{Duration, Instant, LocalDate, LocalDateTime} import java.util - import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.math.BigDecimal.RoundingMode import scala.util.Random -import org.apache.spark.sql.{Column, DataFrame, SparkSession} +import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Expression, XXH64} import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils} -import org.apache.spark.sql.functions.col +import org.apache.spark.sql.functions.{approx_count_distinct, avg, coalesce, col, count, lit, stddev, struct, transform, udf, when} import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.random.XORShiftRandom @@ -79,22 +80,28 @@ class RowLocation(val rowNum: Long, val subRows: Array[Int] = null) { * hash. This makes the generated data correlated for all column/child columns. * @param tableNum a unique ID for the table this is a part of. * @param columnNum the location of the column in the data being generated + * @param substringNum the location of the substring column * @param correlatedKeyGroup the correlated key group this column is a part of, if any. */ -case class ColumnLocation(tableNum: Int, columnNum: Int, correlatedKeyGroup: Option[Long] = None) { - def forNextColumn(): ColumnLocation = ColumnLocation(tableNum, columnNum + 1) +case class ColumnLocation(tableNum: Int, + columnNum: Int, + substringNum: Int, + correlatedKeyGroup: Option[Long] = None) { + def forNextColumn(): ColumnLocation = ColumnLocation(tableNum, columnNum + 1, 0) + def forNextSubstring: ColumnLocation = ColumnLocation(tableNum, columnNum, substringNum + 1) /** * Create a new ColumnLocation that is specifically for a given key group */ def forCorrelatedKeyGroup(keyGroup: Long): ColumnLocation = - ColumnLocation(tableNum, columnNum, Some(keyGroup)) + ColumnLocation(tableNum, columnNum, substringNum, Some(keyGroup)) /** * Hash the location into a single long value. */ - lazy val hashLoc: Long = XXH64.hashLong(tableNum, correlatedKeyGroup.getOrElse(columnNum)) + lazy val hashLoc: Long = XXH64.hashLong(tableNum, + correlatedKeyGroup.getOrElse(XXH64.hashLong(columnNum, substringNum))) } /** @@ -115,6 +122,9 @@ case class ColumnConf(columnLoc: ColumnLocation, def forNextColumn(nullable: Boolean): ColumnConf = ColumnConf(columnLoc.forNextColumn(), nullable, numTableRows) + def forNextSubstring: ColumnConf = + ColumnConf(columnLoc.forNextSubstring, nullable = true, numTableRows) + /** * Create a new configuration based on this, but for a given correlated key group. */ @@ -303,6 +313,23 @@ case class VarLengthGeneratorFunction(minLength: Int, maxLength: Int) extends } } +case class StdDevLengthGen(mean: Double, + stdDev: Double, + mapping: LocationToSeedMapping = null) extends + LengthGeneratorFunction { + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): LengthGeneratorFunction = + StdDevLengthGen(mean, stdDev, mapping) + + override def apply(rowLoc: RowLocation): Int = { + val r = DataGen.getRandomFor(rowLoc, mapping) + val g = r.nextGaussian() // g has a mean of 0 and a stddev of 1.0 + val adjusted = mean + (g * stdDev) + // If the range of seed is too small compared to the stddev and mean we will + // end up with an invalid distribution, but they asked for it. + math.max(0, math.round(adjusted).toInt) + } +} + /** * Generate nulls with a given probability. * @param prob 0.0 to 1.0 for how often nulls should appear in the output. @@ -562,11 +589,8 @@ case class DataGenExpr(child: Expression, } } -/** - * Base class for generating a column/sub-column. This holds configuration for the column, - * and handles what is needed to convert it into GeneratorFunction - */ -abstract class DataGen(var conf: ColumnConf, +abstract class CommonDataGen( + var conf: ColumnConf, defaultValueRange: Option[(Any, Any)], var seedMapping: LocationToSeedMapping = FlatDistribution(), var nullMapping: LocationToSeedMapping = FlatDistribution(), @@ -576,26 +600,25 @@ abstract class DataGen(var conf: ColumnConf, protected var valueRange: Option[(Any, Any)] = defaultValueRange /** - * Set a value range for this data gen. + * Set a value range */ - def setValueRange(min: Any, max: Any): DataGen = { + def setValueRange(min: Any, max: Any): CommonDataGen = { valueRange = Some((min, max)) this } /** - * Set a custom GeneratorFunction to use for this column. + * Set a custom GeneratorFunction */ - def setValueGen(f: GeneratorFunction): DataGen = { + def setValueGen(f: GeneratorFunction): CommonDataGen = { userProvidedValueGen = Some(f) this } /** - * Set a NullGeneratorFunction for this column. This will not be used - * if the column is not nullable. + * Set a NullGeneratorFunction */ - def setNullGen(f: NullGeneratorFunction): DataGen = { + def setNullGen(f: NullGeneratorFunction): CommonDataGen = { this.userProvidedNullGen = Some(f) this } @@ -604,12 +627,12 @@ abstract class DataGen(var conf: ColumnConf, * Set the probability of a null appearing in the output. The probability should be * 0.0 to 1.0. */ - def setNullProbability(probability: Double): DataGen = { + def setNullProbability(probability: Double): CommonDataGen = { this.userProvidedNullGen = Some(NullProbabilityGenerationFunction(probability)) this } - def setNullProbabilityRecursively(probability: Double): DataGen = { + def setNullProbabilityRecursively(probability: Double): CommonDataGen = { this.userProvidedNullGen = Some(NullProbabilityGenerationFunction(probability)) children.foreach { case (_, dataGen) => @@ -621,7 +644,7 @@ abstract class DataGen(var conf: ColumnConf, /** * Set a specific location to seed mapping for the value generation. */ - def setSeedMapping(seedMapping: LocationToSeedMapping): DataGen = { + def setSeedMapping(seedMapping: LocationToSeedMapping): CommonDataGen = { this.seedMapping = seedMapping this } @@ -629,7 +652,7 @@ abstract class DataGen(var conf: ColumnConf, /** * Set a specific location to seed mapping for the null generation. */ - def setNullMapping(nullMapping: LocationToSeedMapping): DataGen = { + def setNullMapping(nullMapping: LocationToSeedMapping): CommonDataGen = { this.nullMapping = nullMapping this } @@ -638,7 +661,7 @@ abstract class DataGen(var conf: ColumnConf, * Set a specific LengthGeneratorFunction to use. This will only be used if * the datatype needs a length. */ - def setLengthGen(lengthGen: LengthGeneratorFunction): DataGen = { + def setLengthGen(lengthGen: LengthGeneratorFunction): CommonDataGen = { this.lengthGen = lengthGen this } @@ -646,25 +669,30 @@ abstract class DataGen(var conf: ColumnConf, /** * Set the length generation to be a fixed length. */ - def setLength(len: Int): DataGen = { + def setLength(len: Int): CommonDataGen = { this.lengthGen = FixedLengthGeneratorFunction(len) this } - def setLength(minLen: Int, maxLen: Int) = { + def setLength(minLen: Int, maxLen: Int): CommonDataGen = { this.lengthGen = VarLengthGeneratorFunction(minLen, maxLen) this } + def setGaussianLength(mean: Double, stdDev: Double): CommonDataGen = { + this.lengthGen = StdDevLengthGen(mean, stdDev) + this + } + /** * Add this column to a specific correlated key group. This should not be * called directly by users. */ def setCorrelatedKeyGroup(keyGroup: Long, - minSeed: Long, maxSeed: Long, - seedMapping: LocationToSeedMapping): DataGen = { + minSeed: Long, maxSeed: Long, + seedMapping: LocationToSeedMapping): CommonDataGen = { conf = conf.forCorrelatedKeyGroup(keyGroup) - .forSeedRange(minSeed, maxSeed) + .forSeedRange(minSeed, maxSeed) this.seedMapping = seedMapping this } @@ -672,7 +700,7 @@ abstract class DataGen(var conf: ColumnConf, /** * Set a range of seed values that should be returned by the LocationToSeedMapping */ - def setSeedRange(min: Long, max: Long): DataGen = { + def setSeedRange(min: Long, max: Long): CommonDataGen = { conf = conf.forSeedRange(min, max) this } @@ -681,7 +709,7 @@ abstract class DataGen(var conf: ColumnConf, * Get the default value generator for this specific data gen. */ protected def getValGen: GeneratorFunction - def children: Seq[(String, DataGen)] + def children: Seq[(String, CommonDataGen)] /** * Get the final ready to use GeneratorFunction for the data generator. @@ -690,8 +718,8 @@ abstract class DataGen(var conf: ColumnConf, val sm = seedMapping.withColumnConf(conf) val lg = lengthGen.withLocationToSeedMapping(sm) var valGen = userProvidedValueGen.getOrElse(getValGen) - .withLocationToSeedMapping(sm) - .withLengthGeneratorFunction(lg) + .withLocationToSeedMapping(sm) + .withLengthGeneratorFunction(lg) valueRange.foreach { case (min, max) => valGen = valGen.withValueRange(min, max) @@ -700,35 +728,75 @@ abstract class DataGen(var conf: ColumnConf, val nullColConf = conf.forNulls val nm = nullMapping.withColumnConf(nullColConf) userProvidedNullGen.get - .withWrapped(valGen) - .withLocationToSeedMapping(nm) + .withWrapped(valGen) + .withLocationToSeedMapping(nm) } else { valGen } } - /** - * Get the data type for this column - */ - def dataType: DataType - /** * Is this column nullable or not. */ def nullable: Boolean = conf.nullable /** - * Get a child column for a given name, if it has one. + * Get a child for a given name, if it has one. */ - final def apply(name: String): DataGen = { + final def apply(name: String): CommonDataGen = { get(name).getOrElse{ throw new IllegalStateException(s"Could not find a child $name for $this") } } - def get(name: String): Option[DataGen] = None + def get(name: String): Option[CommonDataGen] = None +} + + +/** + * Base class for generating a column/sub-column. This holds configuration + * for the column, and handles what is needed to convert it into GeneratorFunction + */ +abstract class DataGen( + conf: ColumnConf, + defaultValueRange: Option[(Any, Any)], + seedMapping: LocationToSeedMapping = FlatDistribution(), + nullMapping: LocationToSeedMapping = FlatDistribution(), + lengthGen: LengthGeneratorFunction = FixedLengthGeneratorFunction(10)) extends + CommonDataGen(conf, defaultValueRange, seedMapping, nullMapping, lengthGen) { + + /** + * Get the data type for this column + */ + def dataType: DataType + + override def get(name: String): Option[DataGen] = None + + def getSubstringGen: Option[SubstringDataGen] = None + + def substringGen: SubstringDataGen = + getSubstringGen.getOrElse( + throw new IllegalArgumentException("substring data gen was not set")) + + def setSubstringGen(f : ColumnConf => SubstringDataGen): Unit = + setSubstringGen(Option(f(conf.forNextSubstring))) + + def setSubstringGen(subgen: Option[SubstringDataGen]): Unit = + throw new IllegalArgumentException("substring data gens can only be set for a STRING") } +/** + * Base class for generating a sub-string. This holds configuration + * for the substring, and handles what is needed to convert it into a GeneratorFunction + */ +abstract class SubstringDataGen( + conf: ColumnConf, + defaultValueRange: Option[(Any, Any)], + seedMapping: LocationToSeedMapping = FlatDistribution(), + nullMapping: LocationToSeedMapping = FlatDistribution(), + lengthGen: LengthGeneratorFunction = FixedLengthGeneratorFunction(10)) extends + CommonDataGen(conf, defaultValueRange, seedMapping, nullMapping, lengthGen) {} + /** * A special GeneratorFunction that just returns the computed seed. This is helpful for * debugging distributions or if you want long values without any abstraction in between. @@ -1494,155 +1562,866 @@ class FloatGen(conf: ColumnConf, defaultValueRange: Option[(Any, Any)]) override def children: Seq[(String, DataGen)] = Seq.empty } -trait JSONType { - def appendRandomValue(sb: StringBuilder, - index: Int, - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - depth: Int, - maxDepth: Int, - r: Random): Unit -} +case class JsonPathElement(name: String, is_array: Boolean) +case class JsonLevel(path: Array[JsonPathElement], data_type: String, length: Int, value: String) {} + +object JsonColumnStats { + private def printHelp(): Unit = { + println("JSON Fingerprinting Tool:") + println("PARAMS: ") + println(" is a path to a Spark dataframe to read in") + println(" is a path in a Spark file system to write out fingerprint data to.") + println() + println("OPTIONS:") + println(" --json= where is the name of a top level String column") + println(" --anon= where is a SEED used to anonymize the JSON keys ") + println(" and column names.") + println(" --input_format= where is parquet or ORC. Defaults to parquet.") + println(" --overwrite to enable overwriting the fingerprint output.") + println(" --debug to enable some debug information to be printed out") + println(" --help to print out this help message") + println() + } + + def main(args: Array[String]): Unit = { + var inputPath = Option.empty[String] + var outputPath = Option.empty[String] + val jsonColumns = ArrayBuffer.empty[String] + var anonSeed = Option.empty[Long] + var debug = false + var argsDone = false + var format = "parquet" + var overwrite = false + + args.foreach { + case a if !argsDone && a.startsWith("--json=") => + jsonColumns += a.substring("--json=".length) + case a if !argsDone && a.startsWith("--anon=") => + anonSeed = Some(a.substring("--anon=".length).toLong) + case a if !argsDone && a.startsWith("--input_format=") => + format = a.substring("--input_format=".length).toLowerCase(java.util.Locale.US) + case "--overwrite" if !argsDone => + overwrite = true + case "--debug" if !argsDone => + debug = true + case "--help" if !argsDone => + printHelp() + System.exit(0) + case "--" if !argsDone => + argsDone = true + case a if !argsDone && a.startsWith("--") => // "--" was covered above already + println(s"ERROR $a is not a supported argument") + printHelp() + System.exit(-1) + case a if inputPath.isEmpty => + inputPath = Some(a) + case a if outputPath.isEmpty => + outputPath = Some(a) + case a => + println(s"ERROR only two arguments are supported. Found $a") + printHelp() + System.exit(-1) + } + if (outputPath.isEmpty) { + println("ERROR both an inputPath and an outputPath are required") + printHelp() + System.exit(-1) + } + + val spark = SparkSession.builder.getOrCreate() + spark.sparkContext.setLogLevel("WARN") + + val df = spark.read.format(format).load(inputPath.get) + jsonColumns.foreach { column => + val fp = fingerPrint(df, df(column), anonSeed) + val name = anonSeed.map(s => anonymizeString(column, s)).getOrElse(column) + val fullOutPath = s"${outputPath.get}/$name" + var writer = fp.write + if (overwrite) { + writer = writer.mode("overwrite") + } + if (debug) { + anonSeed.foreach { s => + println(s"Keys and columns will be anonymized with seed $s") + } + println(s"Writing $column fingerprint to $fullOutPath") + spark.time(writer.parquet(fullOutPath)) + println(s"Wrote ${spark.read.parquet(fullOutPath).count} rows") + spark.read.parquet(fullOutPath).show() + } else { + writer.parquet(fullOutPath) + } + } + } -object JSONType { - def selectType(depth: Int, - maxDepth: Int, - r: Random): JSONType = { - val toSelectFrom = if (depth < maxDepth) { - Seq(QuotedJSONString, JSONLong, JSONDouble, JSONArray, JSONObject) - } else { - Seq(QuotedJSONString, JSONLong, JSONDouble) - } - val index = r.nextInt(toSelectFrom.length) - toSelectFrom(index) - } -} - -object QuotedJSONString extends JSONType { - override def appendRandomValue(sb: StringBuilder, - index: Int, - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - depth: Int, - maxDepth: Int, - r: Random): Unit = { - val strValue = r.nextString(r.nextInt(maxStringLength + 1)) - .replace("\\", "\\\\") - .replace("\"", "\\\"") - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\b", "\\b") - .replace("\f", "\\f") - sb.append('"') - sb.append(strValue) - sb.append('"') - } -} - -object JSONLong extends JSONType { - override def appendRandomValue(sb: StringBuilder, - index: Int, - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - depth: Int, - maxDepth: Int, - r: Random): Unit = { - sb.append(r.nextLong()) - } -} - -object JSONDouble extends JSONType { - override def appendRandomValue(sb: StringBuilder, - index: Int, - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - depth: Int, - maxDepth: Int, - r: Random): Unit = { - sb.append(r.nextDouble() * 4096.0) - } -} - -object JSONArray extends JSONType { - override def appendRandomValue(sb: StringBuilder, - index: Int, - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - depth: Int, - maxDepth: Int, - r: Random): Unit = { - val childType = JSONType.selectType(depth, maxDepth, r) - val length = r.nextInt(maxArrayLength + 1) - sb.append("[") + case class JsonNodeStats(count: Long, meanLen: Double, stdDevLength: Double, dc: Long) + + class JsonNode() { + private val forDataType = + mutable.HashMap[String, (JsonNodeStats, mutable.HashMap[String, JsonNode])]() + + def getChild(name: String, isArray: Boolean): JsonNode = { + val dt = if (isArray) { "ARRAY" } else { "OBJECT" } + val typed = forDataType.getOrElse(dt, + throw new IllegalArgumentException(s"$dt is not a set data type yet.")) + typed._2.getOrElse(name, + throw new IllegalArgumentException(s"$name is not a child when the type is $dt")) + } + + def contains(name: String, isArray: Boolean): Boolean = { + val dt = if (isArray) { "ARRAY" } else { "OBJECT" } + forDataType.get(dt).exists { children => + children._2.contains(name) + } + } + + def addChild(name: String, isArray: Boolean): JsonNode = { + val dt = if (isArray) { "ARRAY" } else { "OBJECT" } + val found = forDataType.getOrElse(dt, + throw new IllegalArgumentException(s"$dt was not already added as a data type")) + if (found._2.contains(name)) { + throw new IllegalArgumentException(s"$dt already has a child named $name") + } + val node = new JsonNode() + found._2.put(name, node) + node + } + + def addChoice(dt: String, stats: JsonNodeStats): Unit = { + if (forDataType.contains(dt)) { + throw new IllegalArgumentException(s"$dt was already added as a data type") + } + forDataType.put(dt, (stats, new mutable.HashMap[String, JsonNode]())) + } + + override def toString: String = { + forDataType.toString() + } + + def totalCount: Long = { + forDataType.values.map{ case (stats, _) => stats.count}.sum + } + + private def makeNoChoiceGenRecursive(dt: String, + children: mutable.HashMap[String, JsonNode], + cc: ColumnConf): (SubstringDataGen, ColumnConf) = { + var c = cc + val ret = dt match { + case "LONG" => new JSONLongGen(c) + case "DOUBLE" => new JSONDoubleGen(c) + case "BOOLEAN" => new JSONBoolGen(c) + case "NULL" => new JSONNullGen(false, c) + case "VALUE_NULL" => new JSONNullGen(true, c) + case "ERROR" => new JSONErrorGen(c) + case "STRING" => new JSONStringGen(c) + case "ARRAY" => + val child = if (children.isEmpty) { + // A corner case, we will just make it a BOOL column and it will be ignored + val tmp = new JSONBoolGen(c) + c = c.forNextSubstring + tmp + } else { + val tmp = children.values.head.makeGenRecursive(c) + c = tmp._2 + tmp._1 + } + new JSONArrayGen(child, c) + case "OBJECT" => + val childGens = if (children.isEmpty) { + Seq.empty + } else { + children.toSeq.map { + case (k, node) => + val tmp = node.makeGenRecursive(c) + c = tmp._2 + (k, tmp._1) + } + } + new JSONObjectGen(childGens, c) + case other => + throw new IllegalArgumentException(s"$other is not a leaf node type") + } + (ret, c.forNextSubstring) + } + + private def makeGenRecursive(cc: ColumnConf): (SubstringDataGen, ColumnConf) = { + var c = cc + // We are going to recursively walk the tree for all of the values. + if (forDataType.size == 1) { + // We don't need a choice at all. This makes it simpler.. + val (dt, (_, children)) = forDataType.head + makeNoChoiceGenRecursive(dt, children, c) + } else { + val totalSum = forDataType.map(f => f._2._1.count).sum.toDouble + var runningSum = 0L + val allChoices = ArrayBuffer[(Double, String, SubstringDataGen)]() + forDataType.foreach { + case (dt, (stats, children)) => + val tmp = makeNoChoiceGenRecursive(dt, children, c) + c = tmp._2 + runningSum += stats.count + allChoices.append((runningSum/totalSum, dt, tmp._1)) + } + + val ret = new JSONChoiceGen(allChoices.toSeq, c) + (ret, c.forNextSubstring) + } + } + + def makeGen(cc: ColumnConf): SubstringDataGen = { + val (ret, _) = makeGenRecursive(cc) + ret + } + + def setStatsSingle(dg: CommonDataGen, + dt: String, + stats: JsonNodeStats, + nullPct: Double): Unit = { + + val includeLength = dt != "OBJECT" && dt != "BOOLEAN" && dt != "NULL" && dt != "VALUE_NULL" + val includeNullPct = nullPct > 0.0 + if (includeLength) { + dg.setGaussianLength(stats.meanLen, stats.stdDevLength) + } + if (includeNullPct) { + dg.setNullProbability(nullPct) + } + dg.setSeedRange(1, stats.dc) + } + + def setStats(dg: CommonDataGen, + parentCount: Option[Long]): Unit = { + // We are going to recursively walk the tree... + if (forDataType.size == 1) { + // We don't need a choice at all. This makes it simpler.. + val (dt, (stats, children)) = forDataType.head + val nullPct = parentCount.map { pc => + (pc - stats.count).toDouble/pc + }.getOrElse(0.0) + setStatsSingle(dg, dt, stats, nullPct) + val myCount = if (dt == "OBJECT") { + Some(totalCount) + } else { + None + } + children.foreach { + case (name, node) => + node.setStats(dg(name), myCount) + } + } else { + // We have choices to make between different types. + // The null percent cannot be calculated for each individual choice + // but is calculated on the group as a whole instead + parentCount.foreach { pc => + val tc = totalCount + val choiceNullPct = (pc - tc).toDouble / pc + if (choiceNullPct > 0.0) { + dg.setNullProbability(choiceNullPct) + } + } + forDataType.foreach { + case (dt, (stats, children)) => + // When there is a choice the name to access it is the data type + val choiceDg = dg(dt) + setStatsSingle(choiceDg, dt, stats, 0.0) + children.foreach { + case (name, node) => + val myCount = if (dt == "OBJECT") { + // Here we only want the count for the OBJECTs + Some(stats.count) + } else { + None + } + node.setStats(choiceDg(name), myCount) + } + } + } + } + } + + private lazy val jsonFactory = new JsonFactoryBuilder() + // The two options below enabled for Hive compatibility + .enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS) + .enable(JsonReadFeature.ALLOW_SINGLE_QUOTES) + .build() + + private def processNext(parser: JsonParser, + currentPath: ArrayBuffer[JsonPathElement], + output: ArrayBuffer[JsonLevel]): Unit = { + parser.currentToken() match { + case JsonToken.START_OBJECT => + parser.nextToken() + while (parser.currentToken() != JsonToken.END_OBJECT) { + processNext(parser, currentPath, output) + } + output.append(JsonLevel(currentPath.toArray, "OBJECT", 0, "")) + parser.nextToken() + case JsonToken.START_ARRAY => + currentPath.append(JsonPathElement("data", is_array = true)) + parser.nextToken() + var length = 0 + while (parser.currentToken() != JsonToken.END_ARRAY) { + length += 1 + processNext(parser, currentPath, output) + } + currentPath.remove(currentPath.length - 1) + output.append(JsonLevel(currentPath.toArray, "ARRAY", length, "")) + parser.nextToken() + case JsonToken.FIELD_NAME => + currentPath.append(JsonPathElement(parser.getCurrentName, is_array = false)) + parser.nextToken() + processNext(parser, currentPath, output) + currentPath.remove(currentPath.length - 1) + case JsonToken.VALUE_NUMBER_INT => + val length = parser.getValueAsString.getBytes("UTF-8").length + output.append(JsonLevel(currentPath.toArray, "LONG", length, parser.getValueAsString)) + parser.nextToken() + case JsonToken.VALUE_NUMBER_FLOAT => + val length = parser.getValueAsString.getBytes("UTF-8").length + output.append(JsonLevel(currentPath.toArray, "DOUBLE", length, parser.getValueAsString)) + parser.nextToken() + case JsonToken.VALUE_TRUE | JsonToken.VALUE_FALSE => + val length = parser.getValueAsString.getBytes("UTF-8").length + output.append(JsonLevel(currentPath.toArray, "BOOLEAN", length, parser.getValueAsString)) + parser.nextToken() + case JsonToken.VALUE_NULL | null => + output.append(JsonLevel(currentPath.toArray, "VALUE_NULL", 4, "NULL")) + parser.nextToken() + case JsonToken.VALUE_STRING => + val length = parser.getValueAsString.getBytes("UTF-8").length + output.append(JsonLevel(currentPath.toArray, "STRING", length, parser.getValueAsString)) + parser.nextToken() + case other => + throw new IllegalStateException(s"DON'T KNOW HOW TO DEAL WITH $other") + } + } + + def jsonStatsUdf(json: String): Array[JsonLevel] = { + val output = new ArrayBuffer[JsonLevel]() + try { + val currentPath = new ArrayBuffer[JsonPathElement]() + if (json == null) { + output.append(JsonLevel(Array.empty, "NULL", 0, "")) + } else { + val parser = jsonFactory.createParser(json) + try { + parser.nextToken() + processNext(parser, currentPath, output) + } finally { + parser.close() + } + } + } catch { + case _: com.fasterxml.jackson.core.JsonParseException => + output.clear() + output.append(JsonLevel(Array.empty, "ERROR", json.getBytes("UTF-8").length, json)) + } + output.toArray + } + + private lazy val extractPaths = udf(json => jsonStatsUdf(json)) + + def anonymizeString(str: String, seed: Long): String = { + val length = str.length + val data = new Array[Byte](length) + val hash = XXH64.hashLong(str.hashCode, seed) + val r = new Random() + r.setSeed(hash) (0 until length).foreach { i => - if (i > 0) { - sb.append(",") + val tmp = r.nextInt(16) + data(i) = (tmp + 'A').toByte + } + new String(data) + } + + private lazy val anonPath = udf((str, seed) => anonymizeString(str, seed)) + + def anonymizeFingerPrint(df: DataFrame, anonSeed: Long): DataFrame = { + df.withColumn("tmp", transform(col("path"), + o => { + val name = o("name") + val isArray = o("is_array") + val anon = anonPath(name, lit(anonSeed)) + val newName = when(isArray, name).otherwise(anon).alias("name") + struct(newName, isArray) + })) + .drop("path").withColumnRenamed("tmp", "path") + .orderBy("path", "dt") + .selectExpr("path", "dt","c","mean_len","stddev_len","distinct","version") + } + + def fingerPrint(df: DataFrame, column: Column, anonymize: Option[Long] = None): DataFrame = { + val ret = df.select(extractPaths(column).alias("paths")) + .selectExpr("explode_outer(paths) as p") + .selectExpr("p.path as path", "p.data_type as dt", "p.length as len", "p.value as value") + .groupBy(col("path"), col("dt")).agg( + count(lit(1)).alias("c"), + avg(col("len")).alias("mean_len"), + coalesce(stddev(col("len")), lit(0.0)).alias("stddev_len"), + approx_count_distinct(col("value")).alias("distinct")) + .orderBy("path", "dt").withColumn("version", lit("0.1")) + .selectExpr("path", "dt","c","mean_len","stddev_len","distinct","version") + + anonymize.map { anonSeed => + anonymizeFingerPrint(ret, anonSeed) + }.getOrElse(ret) + } + + def apply(aggForColumn: DataFrame, genColumn: ColumnGen): Unit = + apply(aggForColumn, genColumn.dataGen) + + private val expectedSchema = StructType.fromDDL( + "path ARRAY>," + + "dt STRING," + + "c BIGINT," + + "mean_len DOUBLE," + + "stddev_len DOUBLE," + + "distinct BIGINT," + + "version STRING") + + def apply(aggForColumn: DataFrame, gen: DataGen): Unit = { + val aggData = aggForColumn.orderBy("path", "dt").collect() + val rootNode: JsonNode = new JsonNode() + assert(aggData.length > 0) + val schema = aggData.head.schema + assert(schema.length == expectedSchema.length) + schema.fields.zip(expectedSchema.fields).foreach { + case(found, expected) => + assert(found.name == expected.name) + // TODO we can worry about the exact types later if we need to + } + assert(aggData.head.getString(6) == "0.1") + aggData.foreach { row => + val fullPath = row.getAs[mutable.WrappedArray[Row]](0) + val parsedPath = fullPath.map(r => (r.getString(0), r.getBoolean(1))).toList + val dt = row.getString(1) + val count = row.getLong(2) + val meanLen = row.getDouble(3) + val stdLen = row.getDouble(4) + val dc = row.getLong(5) + + val stats = JsonNodeStats(count, meanLen, stdLen, dc) + var currentNode = rootNode + // Find everything up to the last path element + if (parsedPath.length > 1) { + parsedPath.slice(0, parsedPath.length - 1).foreach { + case (name, isArray) => + currentNode = currentNode.getChild(name, isArray) + } + } + + if (parsedPath.nonEmpty) { + // For the last path element (that is not the root element) we might need to add it + // as a child + val (name, isArray) = parsedPath.last + if (!currentNode.contains(name, isArray)) { + currentNode.addChild(name, isArray) + } + currentNode = currentNode.getChild(name, isArray) } - childType.appendRandomValue(sb, i, maxStringLength, maxArrayLength, maxObjectLength, - depth + 1, maxDepth, r) + currentNode.addChoice(dt, stats) } - sb.append("]") + + gen.setSubstringGen(cc => rootNode.makeGen(cc)) + rootNode.setStats(gen.substringGen, None) } } -object JSONObject extends JSONType { - override def appendRandomValue(sb: StringBuilder, - index: Int, - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - depth: Int, - maxDepth: Int, - r: Random): Unit = { - val length = r.nextInt(maxObjectLength) + 1 - sb.append("{") - (0 until length).foreach { i => - if (i > 0) { - sb.append(",") + +case class JSONStringGenFunc(lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = { + val len = lengthGen(rowLoc) + val r = DataGen.getRandomFor(rowLoc, mapping) + val buffer = new Array[Byte](len) + var at = 0 + while (at < len) { + // Value range is 32 (Space) to 126 (~) + buffer(at) = (r.nextInt(126 - 31) + 32).toByte + at += 1 + } + val strVal = new String(buffer, 0, len) + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\b", "\\b") + .replace("\f", "\\f") + '"' + strVal + '"' + } + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONStringGenFunc = + JSONStringGenFunc(lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONStringGenFunc = + JSONStringGenFunc(lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONStringGen(conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override protected def getValGen: GeneratorFunction = JSONStringGenFunc() + + override def children: Seq[(String, SubstringDataGen)] = Seq.empty +} + +case class JSONLongGenFunc(lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = { + val len = math.max(lengthGen(rowLoc), 1) // We need at least 1 long for a valid value + val r = DataGen.getRandomFor(rowLoc, mapping) + val buffer = new Array[Byte](len) + var at = 0 + while (at < len) { + if (at == 0) { + // No leading 0's + buffer(at) = (r.nextInt(9) + '1').toByte + } else { + buffer(at) = (r.nextInt(10) + '0').toByte } - sb.append("\"key_") - sb.append(i) - sb.append("_") - sb.append(depth ) - sb.append("\":") - val childType = JSONType.selectType(depth, maxDepth, r) - childType.appendRandomValue(sb, i, maxStringLength, maxArrayLength, maxObjectLength, - depth + 1, maxDepth, r) + at += 1 } - sb.append("}") + new String(buffer, 0, len) } + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONLongGenFunc = + JSONLongGenFunc(lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONLongGenFunc = + JSONLongGenFunc(lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") } -case class JSONGenFunc( - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - maxDepth: Int, - lengthGen: LengthGeneratorFunction = null, - mapping: LocationToSeedMapping = null) extends GeneratorFunction { +class JSONLongGen(conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override protected def getValGen: GeneratorFunction = JSONLongGenFunc() + + override def children: Seq[(String, SubstringDataGen)] = Seq.empty +} + +case class JSONDoubleGenFunc(lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { override def apply(rowLoc: RowLocation): Any = { + val len = math.max(lengthGen(rowLoc), 3) // We have to have at least 3 chars NUM.NUM val r = DataGen.getRandomFor(rowLoc, mapping) - val sb = new StringBuilder() - JSONObject.appendRandomValue(sb, 0, maxStringLength, maxArrayLength, maxObjectLength, - 0, maxDepth, r) - // For now I am going to have some hard coded keys - UTF8String.fromString(sb.toString()) + val beforeLen = if (len == 3) { 1 } else { r.nextInt(len - 3) + 1 } + val buffer = new Array[Byte](len) + var at = 0 + while (at < len) { + if (at == 0) { + // No leading 0's + buffer(at) = (r.nextInt(9) + '1').toByte + } else if (at == beforeLen) { + buffer(at) = '.' + } else { + buffer(at) = (r.nextInt(10) + '0').toByte + } + at += 1 + } + UTF8String.fromBytes(buffer, 0, len) } - override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): GeneratorFunction = - JSONGenFunc(maxStringLength, maxArrayLength, maxObjectLength, maxDepth, lengthGen, mapping) + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONDoubleGenFunc = + JSONDoubleGenFunc(lengthGen, mapping) - override def withLocationToSeedMapping(mapping: LocationToSeedMapping): GeneratorFunction = - JSONGenFunc(maxStringLength, maxArrayLength, maxObjectLength, maxDepth, lengthGen, mapping) + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONDoubleGenFunc = + JSONDoubleGenFunc(lengthGen, mapping) override def withValueRange(min: Any, max: Any): GeneratorFunction = - throw new IllegalArgumentException("value ranges are not supported for strings") + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONDoubleGen(conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override protected def getValGen: GeneratorFunction = JSONDoubleGenFunc() + + override def children: Seq[(String, SubstringDataGen)] = Seq.empty +} + +case class JSONBoolGenFunc(lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = { + val r = DataGen.getRandomFor(rowLoc, mapping) + val ret = if (r.nextBoolean()) "true" else "false" + UTF8String.fromString(ret) + } + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONBoolGenFunc = + JSONBoolGenFunc(lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONBoolGenFunc = + JSONBoolGenFunc(lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONBoolGen(conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override protected def getValGen: GeneratorFunction = JSONBoolGenFunc() + + override def children: Seq[(String, SubstringDataGen)] = Seq.empty +} + +case class JSONNullGenFunc(nullAsString: Boolean, + lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = + if (nullAsString) { + UTF8String.fromString("null") + } else { + null + } + + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONNullGenFunc = + JSONNullGenFunc(nullAsString, lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONNullGenFunc = + JSONNullGenFunc(nullAsString, lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONNullGen(nullAsString: Boolean, + conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override protected def getValGen: GeneratorFunction = JSONNullGenFunc(nullAsString) + + override def children: Seq[(String, SubstringDataGen)] = Seq.empty +} + +case class JSONErrorGenFunc(lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = { + val len = lengthGen(rowLoc) + val r = DataGen.getRandomFor(rowLoc, mapping) + val buffer = new Array[Byte](len) + var at = 0 + while (at < len) { + // Value range is 32 (Space) to 126 (~) + // But it is almost impossible to show up as valid JSON + buffer(at) = (r.nextInt(126 - 31) + 32).toByte + at += 1 + } + UTF8String.fromBytes(buffer, 0, len) + } + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONErrorGenFunc = + JSONErrorGenFunc(lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONErrorGenFunc = + JSONErrorGenFunc(lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONErrorGen(conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override protected def getValGen: GeneratorFunction = JSONErrorGenFunc() + + override def children: Seq[(String, SubstringDataGen)] = Seq.empty +} + +case class JSONArrayGenFunc(child: GeneratorFunction, + lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = { + val len = lengthGen(rowLoc) + val data = new Array[String](len) + val childRowLoc = rowLoc.withNewChild() + var i = 0 + while (i < len) { + childRowLoc.setLastChildIndex(i) + val v = child(childRowLoc) + if (v == null) { + // A null in an array must look like "null" + data(i) = "null" + } else { + data(i) = v.toString + } + i += 1 + } + val ret = data.mkString("[", ",", "]") + UTF8String.fromString(ret) + } + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONArrayGenFunc = + JSONArrayGenFunc(child, lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONArrayGenFunc = + JSONArrayGenFunc(child, lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONArrayGen(child: SubstringDataGen, + conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override def setCorrelatedKeyGroup(keyGroup: Long, + minSeed: Long, maxSeed: Long, + seedMapping: LocationToSeedMapping): SubstringDataGen = { + super.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) + child.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) + this + } + + override protected def getValGen: GeneratorFunction = JSONArrayGenFunc(child.getGen) + + override def get(name: String): Option[SubstringDataGen] = { + if ("data".equalsIgnoreCase(name) || "child".equalsIgnoreCase(name)) { + Some(child) + } else { + None + } + } + + override def children: Seq[(String, SubstringDataGen)] = Seq(("data", child)) +} + +case class JSONObjectGenFunc(childGens: Array[(String, GeneratorFunction)], + lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + override def apply(rowLoc: RowLocation): Any = { + // TODO randomize the order of the children??? + // TODO duplicate child values??? + // The row location does not change for a struct/object + val data = childGens.map { + case (k, gen) => + val key = k.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\b", "\\b") + .replace("\f", "\\f") + val v = gen.apply(rowLoc) + if (v == null) { + "" + } else { + '"' + key + "\":" + v + } + } + val ret = data.filterNot(_.isEmpty).mkString("{",",","}") + UTF8String.fromString(ret) + } + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONObjectGenFunc = + JSONObjectGenFunc(childGens, lengthGen, mapping) + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONObjectGenFunc = + JSONObjectGenFunc(childGens, lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONObjectGen(val children: Seq[(String, SubstringDataGen)], + conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override def setCorrelatedKeyGroup(keyGroup: Long, + minSeed: Long, maxSeed: Long, + seedMapping: LocationToSeedMapping): SubstringDataGen = { + super.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) + children.foreach { + case (_, gen) => + gen.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) + } + this + } + + override def get(name: String): Option[SubstringDataGen] = + children.collectFirst { + case (childName, dataGen) if childName.equalsIgnoreCase(name) => dataGen + } + + override protected def getValGen: GeneratorFunction = { + val childGens = children.map(c => (c._1, c._2.getGen)).toArray + JSONObjectGenFunc(childGens) + } +} + +case class JSONChoiceGenFunc(choices: List[(Double, GeneratorFunction)], + lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + override def apply(rowLoc: RowLocation): Any = { + val r = DataGen.getRandomFor(rowLoc, mapping) + val l = r.nextDouble() + var index = 0 + while (choices(index)._1 < l) { + index += 1 + } + val childRowLoc = rowLoc.withNewChild() + choices(index)._2(childRowLoc) + } + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONChoiceGenFunc = + JSONChoiceGenFunc(choices, lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONChoiceGenFunc = + JSONChoiceGenFunc(choices, lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONChoiceGen(val choices: Seq[(Double, String, SubstringDataGen)], + conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override val children: Seq[(String, SubstringDataGen)] = + choices.map { case (_, name, gen) => (name, gen) } + + override def setCorrelatedKeyGroup(keyGroup: Long, + minSeed: Long, maxSeed: Long, + seedMapping: LocationToSeedMapping): SubstringDataGen = { + super.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) + children.foreach { + case (_, gen) => + gen.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) + } + this + } + + override def get(name: String): Option[SubstringDataGen] = + children.collectFirst { + case (childName, dataGen) if childName.equalsIgnoreCase(name) => dataGen + } + + override protected def getValGen: GeneratorFunction = { + val childGens = choices.map(c => (c._1, c._3.getGen)).toList + JSONChoiceGenFunc(childGens) + } } case class ASCIIGenFunc( @@ -1672,14 +2451,46 @@ case class ASCIIGenFunc( throw new IllegalArgumentException("value ranges are not supported for strings") } -class StringGen(conf: ColumnConf, defaultValueRange: Option[(Any, Any)]) - extends DataGen(conf, defaultValueRange) { +/** + * This is here to wrap the substring gen function so that its length/settings + * are the ones used when generating a string, and not what was set for the string. + */ +case class SubstringGenFunc( + substringGen: GeneratorFunction, + lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = { + substringGen(rowLoc) + } + + // The length and location seed mapping are just ignored for this... + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): GeneratorFunction = + this + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): GeneratorFunction = + this + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for strings") +} + +class StringGen(conf: ColumnConf, + defaultValueRange: Option[(Any, Any)], + var substringDataGen: Option[SubstringDataGen] = None) + extends DataGen(conf, defaultValueRange) { override def dataType: DataType = StringType - override protected def getValGen: GeneratorFunction = ASCIIGenFunc() + override protected def getValGen: GeneratorFunction = + substringDataGen.map(s => SubstringGenFunc(s.getGen)).getOrElse(ASCIIGenFunc()) override def children: Seq[(String, DataGen)] = Seq.empty + + override def setSubstringGen(subgen: Option[SubstringDataGen]): Unit = + substringDataGen = subgen + + override def getSubstringGen: Option[SubstringDataGen] = substringDataGen } case class StructGenFunc(childGens: Array[GeneratorFunction]) extends GeneratorFunction { @@ -1854,7 +2665,6 @@ class MapGen(key: DataGen, override def children: Seq[(String, DataGen)] = Seq(("key", key), ("value", value)) } - object ColumnGen { private def genInternal(rowNumber: Column, dataType: DataType, @@ -1869,8 +2679,8 @@ object ColumnGen { */ class ColumnGen(val dataGen: DataGen) { def setCorrelatedKeyGroup(kg: Long, - minSeed: Long, maxSeed: Long, - seedMapping: LocationToSeedMapping): ColumnGen = { + minSeed: Long, maxSeed: Long, + seedMapping: LocationToSeedMapping): ColumnGen = { dataGen.setCorrelatedKeyGroup(kg, minSeed, maxSeed, seedMapping) this } @@ -1930,6 +2740,11 @@ class ColumnGen(val dataGen: DataGen) { this } + def setGaussianLength(mean: Double, stdDev: Double): ColumnGen = { + dataGen.setGaussianLength(mean, stdDev) + this + } + final def apply(name: String): DataGen = { get(name).getOrElse { throw new IllegalArgumentException(s"$name not a child of $this") @@ -1941,8 +2756,16 @@ class ColumnGen(val dataGen: DataGen) { def gen(rowNumber: Column): Column = { ColumnGen.genInternal(rowNumber, dataGen.dataType, dataGen.nullable, dataGen.getGen) } + + def getSubstring: Option[SubstringDataGen] = dataGen.getSubstringGen + + def substringGen: SubstringDataGen = dataGen.substringGen + + def setSubstringGen(f : ColumnConf => SubstringDataGen): Unit = + dataGen.setSubstringGen(f) } + sealed trait KeyGroupType /** @@ -2192,7 +3015,7 @@ object DBGen { numRows: Long, mapping: OrderedTypeMapping): Seq[(String, ColumnGen)] = { // a bit of a hack with the column num so that we update it before each time... - var conf = ColumnConf(ColumnLocation(tableId, -1), true, numRows) + var conf = ColumnConf(ColumnLocation(tableId, -1, 0), true, numRows) st.toArray.map { sf => if (!mapping.canMap(sf.dataType, mapping)) { throw new IllegalArgumentException(s"$sf is not supported at this time") From 73d76cfade9758f7f47357b59daa1dc5fe2743fa Mon Sep 17 00:00:00 2001 From: Feng Jiang <106386742+Feng-Jiang28@users.noreply.github.com> Date: Thu, 13 Jun 2024 13:23:38 +0800 Subject: [PATCH 30/79] Concat() Exception bug fix (#11039) * concat_null_bug_fix Signed-off-by: fejiang * concat_null_bug_fix Signed-off-by: fejiang * Setting modified Signed-off-by: fejiang * remove comment Signed-off-by: fejiang * concat considered as empty string Signed-off-by: fejiang --------- Signed-off-by: fejiang --- .../org/apache/spark/sql/rapids/collectionOperations.scala | 4 ++-- .../apache/spark/sql/rapids/utils/RapidsTestSettings.scala | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala index 7f0a82517c3..41c2e5e3776 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/collectionOperations.scala @@ -49,8 +49,8 @@ case class GpuConcat(children: Seq[Expression]) extends GpuComplexTypeMergingExp override def columnarEval(batch: ColumnarBatch): GpuColumnVector = { val res = dataType match { - // Explicitly return null for empty concat as Spark, since cuDF doesn't support empty concat. - case dt if children.isEmpty => GpuScalar.from(null, dt) + // in Spark concat() will be considered as an empty string here + case dt if children.isEmpty => GpuScalar("", dt) // For single column concat, we pass the result of child node to avoid extra cuDF call. case _ if children.length == 1 => children.head.columnarEval(batch) case StringType => stringConcat(batch) diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala index ad93c4dd2e9..4cf155041d9 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala @@ -72,7 +72,6 @@ class RapidsTestSettings extends BackendTestSettings { enableSuite[RapidsMathFunctionsSuite] enableSuite[RapidsRegexpExpressionsSuite] enableSuite[RapidsStringExpressionsSuite] - .exclude("concat", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10775")) .exclude("string substring_index function", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10775")) .exclude("SPARK-22498: Concat should not generate codes beyond 64KB", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10775")) .exclude("SPARK-22549: ConcatWs should not generate codes beyond 64KB", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10775")) From cfd8f0023806412b4b0526453cf4798b12fa66d9 Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Thu, 13 Jun 2024 09:07:32 -0500 Subject: [PATCH 31/79] Revert "Add in the ability to fingerprint JSON columns (#11002)" This reverts commit d9686d4dba4afd6e2c061fd2822d40d9a55d88d4. --- .../spark/sql/tests/datagen/bigDataGen.scala | 1181 +++-------------- 1 file changed, 179 insertions(+), 1002 deletions(-) diff --git a/datagen/src/main/scala/org/apache/spark/sql/tests/datagen/bigDataGen.scala b/datagen/src/main/scala/org/apache/spark/sql/tests/datagen/bigDataGen.scala index 14e0d4e0970..91335afe4e6 100644 --- a/datagen/src/main/scala/org/apache/spark/sql/tests/datagen/bigDataGen.scala +++ b/datagen/src/main/scala/org/apache/spark/sql/tests/datagen/bigDataGen.scala @@ -16,22 +16,21 @@ package org.apache.spark.sql.tests.datagen -import com.fasterxml.jackson.core.{JsonFactoryBuilder, JsonParser, JsonToken} -import com.fasterxml.jackson.core.json.JsonReadFeature import java.math.{BigDecimal => JavaBigDecimal} import java.sql.{Date, Timestamp} import java.time.{Duration, Instant, LocalDate, LocalDateTime} import java.util + import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.math.BigDecimal.RoundingMode import scala.util.Random -import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession} +import org.apache.spark.sql.{Column, DataFrame, SparkSession} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Expression, XXH64} import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils} -import org.apache.spark.sql.functions.{approx_count_distinct, avg, coalesce, col, count, lit, stddev, struct, transform, udf, when} +import org.apache.spark.sql.functions.col import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.random.XORShiftRandom @@ -80,28 +79,22 @@ class RowLocation(val rowNum: Long, val subRows: Array[Int] = null) { * hash. This makes the generated data correlated for all column/child columns. * @param tableNum a unique ID for the table this is a part of. * @param columnNum the location of the column in the data being generated - * @param substringNum the location of the substring column * @param correlatedKeyGroup the correlated key group this column is a part of, if any. */ -case class ColumnLocation(tableNum: Int, - columnNum: Int, - substringNum: Int, - correlatedKeyGroup: Option[Long] = None) { - def forNextColumn(): ColumnLocation = ColumnLocation(tableNum, columnNum + 1, 0) +case class ColumnLocation(tableNum: Int, columnNum: Int, correlatedKeyGroup: Option[Long] = None) { + def forNextColumn(): ColumnLocation = ColumnLocation(tableNum, columnNum + 1) - def forNextSubstring: ColumnLocation = ColumnLocation(tableNum, columnNum, substringNum + 1) /** * Create a new ColumnLocation that is specifically for a given key group */ def forCorrelatedKeyGroup(keyGroup: Long): ColumnLocation = - ColumnLocation(tableNum, columnNum, substringNum, Some(keyGroup)) + ColumnLocation(tableNum, columnNum, Some(keyGroup)) /** * Hash the location into a single long value. */ - lazy val hashLoc: Long = XXH64.hashLong(tableNum, - correlatedKeyGroup.getOrElse(XXH64.hashLong(columnNum, substringNum))) + lazy val hashLoc: Long = XXH64.hashLong(tableNum, correlatedKeyGroup.getOrElse(columnNum)) } /** @@ -122,9 +115,6 @@ case class ColumnConf(columnLoc: ColumnLocation, def forNextColumn(nullable: Boolean): ColumnConf = ColumnConf(columnLoc.forNextColumn(), nullable, numTableRows) - def forNextSubstring: ColumnConf = - ColumnConf(columnLoc.forNextSubstring, nullable = true, numTableRows) - /** * Create a new configuration based on this, but for a given correlated key group. */ @@ -313,23 +303,6 @@ case class VarLengthGeneratorFunction(minLength: Int, maxLength: Int) extends } } -case class StdDevLengthGen(mean: Double, - stdDev: Double, - mapping: LocationToSeedMapping = null) extends - LengthGeneratorFunction { - override def withLocationToSeedMapping(mapping: LocationToSeedMapping): LengthGeneratorFunction = - StdDevLengthGen(mean, stdDev, mapping) - - override def apply(rowLoc: RowLocation): Int = { - val r = DataGen.getRandomFor(rowLoc, mapping) - val g = r.nextGaussian() // g has a mean of 0 and a stddev of 1.0 - val adjusted = mean + (g * stdDev) - // If the range of seed is too small compared to the stddev and mean we will - // end up with an invalid distribution, but they asked for it. - math.max(0, math.round(adjusted).toInt) - } -} - /** * Generate nulls with a given probability. * @param prob 0.0 to 1.0 for how often nulls should appear in the output. @@ -589,8 +562,11 @@ case class DataGenExpr(child: Expression, } } -abstract class CommonDataGen( - var conf: ColumnConf, +/** + * Base class for generating a column/sub-column. This holds configuration for the column, + * and handles what is needed to convert it into GeneratorFunction + */ +abstract class DataGen(var conf: ColumnConf, defaultValueRange: Option[(Any, Any)], var seedMapping: LocationToSeedMapping = FlatDistribution(), var nullMapping: LocationToSeedMapping = FlatDistribution(), @@ -600,25 +576,26 @@ abstract class CommonDataGen( protected var valueRange: Option[(Any, Any)] = defaultValueRange /** - * Set a value range + * Set a value range for this data gen. */ - def setValueRange(min: Any, max: Any): CommonDataGen = { + def setValueRange(min: Any, max: Any): DataGen = { valueRange = Some((min, max)) this } /** - * Set a custom GeneratorFunction + * Set a custom GeneratorFunction to use for this column. */ - def setValueGen(f: GeneratorFunction): CommonDataGen = { + def setValueGen(f: GeneratorFunction): DataGen = { userProvidedValueGen = Some(f) this } /** - * Set a NullGeneratorFunction + * Set a NullGeneratorFunction for this column. This will not be used + * if the column is not nullable. */ - def setNullGen(f: NullGeneratorFunction): CommonDataGen = { + def setNullGen(f: NullGeneratorFunction): DataGen = { this.userProvidedNullGen = Some(f) this } @@ -627,12 +604,12 @@ abstract class CommonDataGen( * Set the probability of a null appearing in the output. The probability should be * 0.0 to 1.0. */ - def setNullProbability(probability: Double): CommonDataGen = { + def setNullProbability(probability: Double): DataGen = { this.userProvidedNullGen = Some(NullProbabilityGenerationFunction(probability)) this } - def setNullProbabilityRecursively(probability: Double): CommonDataGen = { + def setNullProbabilityRecursively(probability: Double): DataGen = { this.userProvidedNullGen = Some(NullProbabilityGenerationFunction(probability)) children.foreach { case (_, dataGen) => @@ -644,7 +621,7 @@ abstract class CommonDataGen( /** * Set a specific location to seed mapping for the value generation. */ - def setSeedMapping(seedMapping: LocationToSeedMapping): CommonDataGen = { + def setSeedMapping(seedMapping: LocationToSeedMapping): DataGen = { this.seedMapping = seedMapping this } @@ -652,7 +629,7 @@ abstract class CommonDataGen( /** * Set a specific location to seed mapping for the null generation. */ - def setNullMapping(nullMapping: LocationToSeedMapping): CommonDataGen = { + def setNullMapping(nullMapping: LocationToSeedMapping): DataGen = { this.nullMapping = nullMapping this } @@ -661,7 +638,7 @@ abstract class CommonDataGen( * Set a specific LengthGeneratorFunction to use. This will only be used if * the datatype needs a length. */ - def setLengthGen(lengthGen: LengthGeneratorFunction): CommonDataGen = { + def setLengthGen(lengthGen: LengthGeneratorFunction): DataGen = { this.lengthGen = lengthGen this } @@ -669,30 +646,25 @@ abstract class CommonDataGen( /** * Set the length generation to be a fixed length. */ - def setLength(len: Int): CommonDataGen = { + def setLength(len: Int): DataGen = { this.lengthGen = FixedLengthGeneratorFunction(len) this } - def setLength(minLen: Int, maxLen: Int): CommonDataGen = { + def setLength(minLen: Int, maxLen: Int) = { this.lengthGen = VarLengthGeneratorFunction(minLen, maxLen) this } - def setGaussianLength(mean: Double, stdDev: Double): CommonDataGen = { - this.lengthGen = StdDevLengthGen(mean, stdDev) - this - } - /** * Add this column to a specific correlated key group. This should not be * called directly by users. */ def setCorrelatedKeyGroup(keyGroup: Long, - minSeed: Long, maxSeed: Long, - seedMapping: LocationToSeedMapping): CommonDataGen = { + minSeed: Long, maxSeed: Long, + seedMapping: LocationToSeedMapping): DataGen = { conf = conf.forCorrelatedKeyGroup(keyGroup) - .forSeedRange(minSeed, maxSeed) + .forSeedRange(minSeed, maxSeed) this.seedMapping = seedMapping this } @@ -700,7 +672,7 @@ abstract class CommonDataGen( /** * Set a range of seed values that should be returned by the LocationToSeedMapping */ - def setSeedRange(min: Long, max: Long): CommonDataGen = { + def setSeedRange(min: Long, max: Long): DataGen = { conf = conf.forSeedRange(min, max) this } @@ -709,7 +681,7 @@ abstract class CommonDataGen( * Get the default value generator for this specific data gen. */ protected def getValGen: GeneratorFunction - def children: Seq[(String, CommonDataGen)] + def children: Seq[(String, DataGen)] /** * Get the final ready to use GeneratorFunction for the data generator. @@ -718,8 +690,8 @@ abstract class CommonDataGen( val sm = seedMapping.withColumnConf(conf) val lg = lengthGen.withLocationToSeedMapping(sm) var valGen = userProvidedValueGen.getOrElse(getValGen) - .withLocationToSeedMapping(sm) - .withLengthGeneratorFunction(lg) + .withLocationToSeedMapping(sm) + .withLengthGeneratorFunction(lg) valueRange.foreach { case (min, max) => valGen = valGen.withValueRange(min, max) @@ -728,75 +700,35 @@ abstract class CommonDataGen( val nullColConf = conf.forNulls val nm = nullMapping.withColumnConf(nullColConf) userProvidedNullGen.get - .withWrapped(valGen) - .withLocationToSeedMapping(nm) + .withWrapped(valGen) + .withLocationToSeedMapping(nm) } else { valGen } } + /** + * Get the data type for this column + */ + def dataType: DataType + /** * Is this column nullable or not. */ def nullable: Boolean = conf.nullable /** - * Get a child for a given name, if it has one. + * Get a child column for a given name, if it has one. */ - final def apply(name: String): CommonDataGen = { + final def apply(name: String): DataGen = { get(name).getOrElse{ throw new IllegalStateException(s"Could not find a child $name for $this") } } - def get(name: String): Option[CommonDataGen] = None -} - - -/** - * Base class for generating a column/sub-column. This holds configuration - * for the column, and handles what is needed to convert it into GeneratorFunction - */ -abstract class DataGen( - conf: ColumnConf, - defaultValueRange: Option[(Any, Any)], - seedMapping: LocationToSeedMapping = FlatDistribution(), - nullMapping: LocationToSeedMapping = FlatDistribution(), - lengthGen: LengthGeneratorFunction = FixedLengthGeneratorFunction(10)) extends - CommonDataGen(conf, defaultValueRange, seedMapping, nullMapping, lengthGen) { - - /** - * Get the data type for this column - */ - def dataType: DataType - - override def get(name: String): Option[DataGen] = None - - def getSubstringGen: Option[SubstringDataGen] = None - - def substringGen: SubstringDataGen = - getSubstringGen.getOrElse( - throw new IllegalArgumentException("substring data gen was not set")) - - def setSubstringGen(f : ColumnConf => SubstringDataGen): Unit = - setSubstringGen(Option(f(conf.forNextSubstring))) - - def setSubstringGen(subgen: Option[SubstringDataGen]): Unit = - throw new IllegalArgumentException("substring data gens can only be set for a STRING") + def get(name: String): Option[DataGen] = None } -/** - * Base class for generating a sub-string. This holds configuration - * for the substring, and handles what is needed to convert it into a GeneratorFunction - */ -abstract class SubstringDataGen( - conf: ColumnConf, - defaultValueRange: Option[(Any, Any)], - seedMapping: LocationToSeedMapping = FlatDistribution(), - nullMapping: LocationToSeedMapping = FlatDistribution(), - lengthGen: LengthGeneratorFunction = FixedLengthGeneratorFunction(10)) extends - CommonDataGen(conf, defaultValueRange, seedMapping, nullMapping, lengthGen) {} - /** * A special GeneratorFunction that just returns the computed seed. This is helpful for * debugging distributions or if you want long values without any abstraction in between. @@ -1562,866 +1494,155 @@ class FloatGen(conf: ColumnConf, defaultValueRange: Option[(Any, Any)]) override def children: Seq[(String, DataGen)] = Seq.empty } -case class JsonPathElement(name: String, is_array: Boolean) -case class JsonLevel(path: Array[JsonPathElement], data_type: String, length: Int, value: String) {} - -object JsonColumnStats { - private def printHelp(): Unit = { - println("JSON Fingerprinting Tool:") - println("PARAMS: ") - println(" is a path to a Spark dataframe to read in") - println(" is a path in a Spark file system to write out fingerprint data to.") - println() - println("OPTIONS:") - println(" --json= where is the name of a top level String column") - println(" --anon= where is a SEED used to anonymize the JSON keys ") - println(" and column names.") - println(" --input_format= where is parquet or ORC. Defaults to parquet.") - println(" --overwrite to enable overwriting the fingerprint output.") - println(" --debug to enable some debug information to be printed out") - println(" --help to print out this help message") - println() - } - - def main(args: Array[String]): Unit = { - var inputPath = Option.empty[String] - var outputPath = Option.empty[String] - val jsonColumns = ArrayBuffer.empty[String] - var anonSeed = Option.empty[Long] - var debug = false - var argsDone = false - var format = "parquet" - var overwrite = false - - args.foreach { - case a if !argsDone && a.startsWith("--json=") => - jsonColumns += a.substring("--json=".length) - case a if !argsDone && a.startsWith("--anon=") => - anonSeed = Some(a.substring("--anon=".length).toLong) - case a if !argsDone && a.startsWith("--input_format=") => - format = a.substring("--input_format=".length).toLowerCase(java.util.Locale.US) - case "--overwrite" if !argsDone => - overwrite = true - case "--debug" if !argsDone => - debug = true - case "--help" if !argsDone => - printHelp() - System.exit(0) - case "--" if !argsDone => - argsDone = true - case a if !argsDone && a.startsWith("--") => // "--" was covered above already - println(s"ERROR $a is not a supported argument") - printHelp() - System.exit(-1) - case a if inputPath.isEmpty => - inputPath = Some(a) - case a if outputPath.isEmpty => - outputPath = Some(a) - case a => - println(s"ERROR only two arguments are supported. Found $a") - printHelp() - System.exit(-1) - } - if (outputPath.isEmpty) { - println("ERROR both an inputPath and an outputPath are required") - printHelp() - System.exit(-1) - } - - val spark = SparkSession.builder.getOrCreate() - spark.sparkContext.setLogLevel("WARN") - - val df = spark.read.format(format).load(inputPath.get) - jsonColumns.foreach { column => - val fp = fingerPrint(df, df(column), anonSeed) - val name = anonSeed.map(s => anonymizeString(column, s)).getOrElse(column) - val fullOutPath = s"${outputPath.get}/$name" - var writer = fp.write - if (overwrite) { - writer = writer.mode("overwrite") - } - if (debug) { - anonSeed.foreach { s => - println(s"Keys and columns will be anonymized with seed $s") - } - println(s"Writing $column fingerprint to $fullOutPath") - spark.time(writer.parquet(fullOutPath)) - println(s"Wrote ${spark.read.parquet(fullOutPath).count} rows") - spark.read.parquet(fullOutPath).show() - } else { - writer.parquet(fullOutPath) - } - } - } - - case class JsonNodeStats(count: Long, meanLen: Double, stdDevLength: Double, dc: Long) - - class JsonNode() { - private val forDataType = - mutable.HashMap[String, (JsonNodeStats, mutable.HashMap[String, JsonNode])]() - - def getChild(name: String, isArray: Boolean): JsonNode = { - val dt = if (isArray) { "ARRAY" } else { "OBJECT" } - val typed = forDataType.getOrElse(dt, - throw new IllegalArgumentException(s"$dt is not a set data type yet.")) - typed._2.getOrElse(name, - throw new IllegalArgumentException(s"$name is not a child when the type is $dt")) - } - - def contains(name: String, isArray: Boolean): Boolean = { - val dt = if (isArray) { "ARRAY" } else { "OBJECT" } - forDataType.get(dt).exists { children => - children._2.contains(name) - } - } - - def addChild(name: String, isArray: Boolean): JsonNode = { - val dt = if (isArray) { "ARRAY" } else { "OBJECT" } - val found = forDataType.getOrElse(dt, - throw new IllegalArgumentException(s"$dt was not already added as a data type")) - if (found._2.contains(name)) { - throw new IllegalArgumentException(s"$dt already has a child named $name") - } - val node = new JsonNode() - found._2.put(name, node) - node - } - - def addChoice(dt: String, stats: JsonNodeStats): Unit = { - if (forDataType.contains(dt)) { - throw new IllegalArgumentException(s"$dt was already added as a data type") - } - forDataType.put(dt, (stats, new mutable.HashMap[String, JsonNode]())) - } - - override def toString: String = { - forDataType.toString() - } - - def totalCount: Long = { - forDataType.values.map{ case (stats, _) => stats.count}.sum - } - - private def makeNoChoiceGenRecursive(dt: String, - children: mutable.HashMap[String, JsonNode], - cc: ColumnConf): (SubstringDataGen, ColumnConf) = { - var c = cc - val ret = dt match { - case "LONG" => new JSONLongGen(c) - case "DOUBLE" => new JSONDoubleGen(c) - case "BOOLEAN" => new JSONBoolGen(c) - case "NULL" => new JSONNullGen(false, c) - case "VALUE_NULL" => new JSONNullGen(true, c) - case "ERROR" => new JSONErrorGen(c) - case "STRING" => new JSONStringGen(c) - case "ARRAY" => - val child = if (children.isEmpty) { - // A corner case, we will just make it a BOOL column and it will be ignored - val tmp = new JSONBoolGen(c) - c = c.forNextSubstring - tmp - } else { - val tmp = children.values.head.makeGenRecursive(c) - c = tmp._2 - tmp._1 - } - new JSONArrayGen(child, c) - case "OBJECT" => - val childGens = if (children.isEmpty) { - Seq.empty - } else { - children.toSeq.map { - case (k, node) => - val tmp = node.makeGenRecursive(c) - c = tmp._2 - (k, tmp._1) - } - } - new JSONObjectGen(childGens, c) - case other => - throw new IllegalArgumentException(s"$other is not a leaf node type") - } - (ret, c.forNextSubstring) - } - - private def makeGenRecursive(cc: ColumnConf): (SubstringDataGen, ColumnConf) = { - var c = cc - // We are going to recursively walk the tree for all of the values. - if (forDataType.size == 1) { - // We don't need a choice at all. This makes it simpler.. - val (dt, (_, children)) = forDataType.head - makeNoChoiceGenRecursive(dt, children, c) - } else { - val totalSum = forDataType.map(f => f._2._1.count).sum.toDouble - var runningSum = 0L - val allChoices = ArrayBuffer[(Double, String, SubstringDataGen)]() - forDataType.foreach { - case (dt, (stats, children)) => - val tmp = makeNoChoiceGenRecursive(dt, children, c) - c = tmp._2 - runningSum += stats.count - allChoices.append((runningSum/totalSum, dt, tmp._1)) - } - - val ret = new JSONChoiceGen(allChoices.toSeq, c) - (ret, c.forNextSubstring) - } - } - - def makeGen(cc: ColumnConf): SubstringDataGen = { - val (ret, _) = makeGenRecursive(cc) - ret - } - - def setStatsSingle(dg: CommonDataGen, - dt: String, - stats: JsonNodeStats, - nullPct: Double): Unit = { - - val includeLength = dt != "OBJECT" && dt != "BOOLEAN" && dt != "NULL" && dt != "VALUE_NULL" - val includeNullPct = nullPct > 0.0 - if (includeLength) { - dg.setGaussianLength(stats.meanLen, stats.stdDevLength) - } - if (includeNullPct) { - dg.setNullProbability(nullPct) - } - dg.setSeedRange(1, stats.dc) - } - - def setStats(dg: CommonDataGen, - parentCount: Option[Long]): Unit = { - // We are going to recursively walk the tree... - if (forDataType.size == 1) { - // We don't need a choice at all. This makes it simpler.. - val (dt, (stats, children)) = forDataType.head - val nullPct = parentCount.map { pc => - (pc - stats.count).toDouble/pc - }.getOrElse(0.0) - setStatsSingle(dg, dt, stats, nullPct) - val myCount = if (dt == "OBJECT") { - Some(totalCount) - } else { - None - } - children.foreach { - case (name, node) => - node.setStats(dg(name), myCount) - } - } else { - // We have choices to make between different types. - // The null percent cannot be calculated for each individual choice - // but is calculated on the group as a whole instead - parentCount.foreach { pc => - val tc = totalCount - val choiceNullPct = (pc - tc).toDouble / pc - if (choiceNullPct > 0.0) { - dg.setNullProbability(choiceNullPct) - } - } - forDataType.foreach { - case (dt, (stats, children)) => - // When there is a choice the name to access it is the data type - val choiceDg = dg(dt) - setStatsSingle(choiceDg, dt, stats, 0.0) - children.foreach { - case (name, node) => - val myCount = if (dt == "OBJECT") { - // Here we only want the count for the OBJECTs - Some(stats.count) - } else { - None - } - node.setStats(choiceDg(name), myCount) - } - } - } - } - } - - private lazy val jsonFactory = new JsonFactoryBuilder() - // The two options below enabled for Hive compatibility - .enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS) - .enable(JsonReadFeature.ALLOW_SINGLE_QUOTES) - .build() - - private def processNext(parser: JsonParser, - currentPath: ArrayBuffer[JsonPathElement], - output: ArrayBuffer[JsonLevel]): Unit = { - parser.currentToken() match { - case JsonToken.START_OBJECT => - parser.nextToken() - while (parser.currentToken() != JsonToken.END_OBJECT) { - processNext(parser, currentPath, output) - } - output.append(JsonLevel(currentPath.toArray, "OBJECT", 0, "")) - parser.nextToken() - case JsonToken.START_ARRAY => - currentPath.append(JsonPathElement("data", is_array = true)) - parser.nextToken() - var length = 0 - while (parser.currentToken() != JsonToken.END_ARRAY) { - length += 1 - processNext(parser, currentPath, output) - } - currentPath.remove(currentPath.length - 1) - output.append(JsonLevel(currentPath.toArray, "ARRAY", length, "")) - parser.nextToken() - case JsonToken.FIELD_NAME => - currentPath.append(JsonPathElement(parser.getCurrentName, is_array = false)) - parser.nextToken() - processNext(parser, currentPath, output) - currentPath.remove(currentPath.length - 1) - case JsonToken.VALUE_NUMBER_INT => - val length = parser.getValueAsString.getBytes("UTF-8").length - output.append(JsonLevel(currentPath.toArray, "LONG", length, parser.getValueAsString)) - parser.nextToken() - case JsonToken.VALUE_NUMBER_FLOAT => - val length = parser.getValueAsString.getBytes("UTF-8").length - output.append(JsonLevel(currentPath.toArray, "DOUBLE", length, parser.getValueAsString)) - parser.nextToken() - case JsonToken.VALUE_TRUE | JsonToken.VALUE_FALSE => - val length = parser.getValueAsString.getBytes("UTF-8").length - output.append(JsonLevel(currentPath.toArray, "BOOLEAN", length, parser.getValueAsString)) - parser.nextToken() - case JsonToken.VALUE_NULL | null => - output.append(JsonLevel(currentPath.toArray, "VALUE_NULL", 4, "NULL")) - parser.nextToken() - case JsonToken.VALUE_STRING => - val length = parser.getValueAsString.getBytes("UTF-8").length - output.append(JsonLevel(currentPath.toArray, "STRING", length, parser.getValueAsString)) - parser.nextToken() - case other => - throw new IllegalStateException(s"DON'T KNOW HOW TO DEAL WITH $other") - } - } - - def jsonStatsUdf(json: String): Array[JsonLevel] = { - val output = new ArrayBuffer[JsonLevel]() - try { - val currentPath = new ArrayBuffer[JsonPathElement]() - if (json == null) { - output.append(JsonLevel(Array.empty, "NULL", 0, "")) - } else { - val parser = jsonFactory.createParser(json) - try { - parser.nextToken() - processNext(parser, currentPath, output) - } finally { - parser.close() - } - } - } catch { - case _: com.fasterxml.jackson.core.JsonParseException => - output.clear() - output.append(JsonLevel(Array.empty, "ERROR", json.getBytes("UTF-8").length, json)) - } - output.toArray - } - - private lazy val extractPaths = udf(json => jsonStatsUdf(json)) - - def anonymizeString(str: String, seed: Long): String = { - val length = str.length - val data = new Array[Byte](length) - val hash = XXH64.hashLong(str.hashCode, seed) - val r = new Random() - r.setSeed(hash) - (0 until length).foreach { i => - val tmp = r.nextInt(16) - data(i) = (tmp + 'A').toByte - } - new String(data) - } - - private lazy val anonPath = udf((str, seed) => anonymizeString(str, seed)) - - def anonymizeFingerPrint(df: DataFrame, anonSeed: Long): DataFrame = { - df.withColumn("tmp", transform(col("path"), - o => { - val name = o("name") - val isArray = o("is_array") - val anon = anonPath(name, lit(anonSeed)) - val newName = when(isArray, name).otherwise(anon).alias("name") - struct(newName, isArray) - })) - .drop("path").withColumnRenamed("tmp", "path") - .orderBy("path", "dt") - .selectExpr("path", "dt","c","mean_len","stddev_len","distinct","version") - } - - def fingerPrint(df: DataFrame, column: Column, anonymize: Option[Long] = None): DataFrame = { - val ret = df.select(extractPaths(column).alias("paths")) - .selectExpr("explode_outer(paths) as p") - .selectExpr("p.path as path", "p.data_type as dt", "p.length as len", "p.value as value") - .groupBy(col("path"), col("dt")).agg( - count(lit(1)).alias("c"), - avg(col("len")).alias("mean_len"), - coalesce(stddev(col("len")), lit(0.0)).alias("stddev_len"), - approx_count_distinct(col("value")).alias("distinct")) - .orderBy("path", "dt").withColumn("version", lit("0.1")) - .selectExpr("path", "dt","c","mean_len","stddev_len","distinct","version") - - anonymize.map { anonSeed => - anonymizeFingerPrint(ret, anonSeed) - }.getOrElse(ret) - } - - def apply(aggForColumn: DataFrame, genColumn: ColumnGen): Unit = - apply(aggForColumn, genColumn.dataGen) - - private val expectedSchema = StructType.fromDDL( - "path ARRAY>," + - "dt STRING," + - "c BIGINT," + - "mean_len DOUBLE," + - "stddev_len DOUBLE," + - "distinct BIGINT," + - "version STRING") - - def apply(aggForColumn: DataFrame, gen: DataGen): Unit = { - val aggData = aggForColumn.orderBy("path", "dt").collect() - val rootNode: JsonNode = new JsonNode() - assert(aggData.length > 0) - val schema = aggData.head.schema - assert(schema.length == expectedSchema.length) - schema.fields.zip(expectedSchema.fields).foreach { - case(found, expected) => - assert(found.name == expected.name) - // TODO we can worry about the exact types later if we need to - } - assert(aggData.head.getString(6) == "0.1") - aggData.foreach { row => - val fullPath = row.getAs[mutable.WrappedArray[Row]](0) - val parsedPath = fullPath.map(r => (r.getString(0), r.getBoolean(1))).toList - val dt = row.getString(1) - val count = row.getLong(2) - val meanLen = row.getDouble(3) - val stdLen = row.getDouble(4) - val dc = row.getLong(5) - - val stats = JsonNodeStats(count, meanLen, stdLen, dc) - var currentNode = rootNode - // Find everything up to the last path element - if (parsedPath.length > 1) { - parsedPath.slice(0, parsedPath.length - 1).foreach { - case (name, isArray) => - currentNode = currentNode.getChild(name, isArray) - } - } - - if (parsedPath.nonEmpty) { - // For the last path element (that is not the root element) we might need to add it - // as a child - val (name, isArray) = parsedPath.last - if (!currentNode.contains(name, isArray)) { - currentNode.addChild(name, isArray) - } - currentNode = currentNode.getChild(name, isArray) - } - currentNode.addChoice(dt, stats) - } - - gen.setSubstringGen(cc => rootNode.makeGen(cc)) - rootNode.setStats(gen.substringGen, None) - } -} - - -case class JSONStringGenFunc(lengthGen: LengthGeneratorFunction = null, - mapping: LocationToSeedMapping = null) extends GeneratorFunction { - - override def apply(rowLoc: RowLocation): Any = { - val len = lengthGen(rowLoc) - val r = DataGen.getRandomFor(rowLoc, mapping) - val buffer = new Array[Byte](len) - var at = 0 - while (at < len) { - // Value range is 32 (Space) to 126 (~) - buffer(at) = (r.nextInt(126 - 31) + 32).toByte - at += 1 - } - val strVal = new String(buffer, 0, len) - .replace("\\", "\\\\") - .replace("\"", "\\\"") - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\b", "\\b") - .replace("\f", "\\f") - '"' + strVal + '"' - } - - override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONStringGenFunc = - JSONStringGenFunc(lengthGen, mapping) - - override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONStringGenFunc = - JSONStringGenFunc(lengthGen, mapping) - - override def withValueRange(min: Any, max: Any): GeneratorFunction = - throw new IllegalArgumentException("value ranges are not supported for JSON") -} - -class JSONStringGen(conf: ColumnConf, - defaultValueRange: Option[(Any, Any)] = None) - extends SubstringDataGen(conf, defaultValueRange) { - - override protected def getValGen: GeneratorFunction = JSONStringGenFunc() - - override def children: Seq[(String, SubstringDataGen)] = Seq.empty -} - -case class JSONLongGenFunc(lengthGen: LengthGeneratorFunction = null, - mapping: LocationToSeedMapping = null) extends GeneratorFunction { - - override def apply(rowLoc: RowLocation): Any = { - val len = math.max(lengthGen(rowLoc), 1) // We need at least 1 long for a valid value - val r = DataGen.getRandomFor(rowLoc, mapping) - val buffer = new Array[Byte](len) - var at = 0 - while (at < len) { - if (at == 0) { - // No leading 0's - buffer(at) = (r.nextInt(9) + '1').toByte - } else { - buffer(at) = (r.nextInt(10) + '0').toByte - } - at += 1 - } - new String(buffer, 0, len) - } - - override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONLongGenFunc = - JSONLongGenFunc(lengthGen, mapping) - - override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONLongGenFunc = - JSONLongGenFunc(lengthGen, mapping) - - override def withValueRange(min: Any, max: Any): GeneratorFunction = - throw new IllegalArgumentException("value ranges are not supported for JSON") -} - -class JSONLongGen(conf: ColumnConf, - defaultValueRange: Option[(Any, Any)] = None) - extends SubstringDataGen(conf, defaultValueRange) { - - override protected def getValGen: GeneratorFunction = JSONLongGenFunc() - - override def children: Seq[(String, SubstringDataGen)] = Seq.empty -} - -case class JSONDoubleGenFunc(lengthGen: LengthGeneratorFunction = null, - mapping: LocationToSeedMapping = null) extends GeneratorFunction { - - override def apply(rowLoc: RowLocation): Any = { - val len = math.max(lengthGen(rowLoc), 3) // We have to have at least 3 chars NUM.NUM - val r = DataGen.getRandomFor(rowLoc, mapping) - val beforeLen = if (len == 3) { 1 } else { r.nextInt(len - 3) + 1 } - val buffer = new Array[Byte](len) - var at = 0 - while (at < len) { - if (at == 0) { - // No leading 0's - buffer(at) = (r.nextInt(9) + '1').toByte - } else if (at == beforeLen) { - buffer(at) = '.' - } else { - buffer(at) = (r.nextInt(10) + '0').toByte - } - at += 1 - } - UTF8String.fromBytes(buffer, 0, len) - } - - override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONDoubleGenFunc = - JSONDoubleGenFunc(lengthGen, mapping) - - override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONDoubleGenFunc = - JSONDoubleGenFunc(lengthGen, mapping) - - override def withValueRange(min: Any, max: Any): GeneratorFunction = - throw new IllegalArgumentException("value ranges are not supported for JSON") -} - -class JSONDoubleGen(conf: ColumnConf, - defaultValueRange: Option[(Any, Any)] = None) - extends SubstringDataGen(conf, defaultValueRange) { - - override protected def getValGen: GeneratorFunction = JSONDoubleGenFunc() - - override def children: Seq[(String, SubstringDataGen)] = Seq.empty +trait JSONType { + def appendRandomValue(sb: StringBuilder, + index: Int, + maxStringLength: Int, + maxArrayLength: Int, + maxObjectLength: Int, + depth: Int, + maxDepth: Int, + r: Random): Unit } -case class JSONBoolGenFunc(lengthGen: LengthGeneratorFunction = null, - mapping: LocationToSeedMapping = null) extends GeneratorFunction { - - override def apply(rowLoc: RowLocation): Any = { - val r = DataGen.getRandomFor(rowLoc, mapping) - val ret = if (r.nextBoolean()) "true" else "false" - UTF8String.fromString(ret) - } - - override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONBoolGenFunc = - JSONBoolGenFunc(lengthGen, mapping) - - override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONBoolGenFunc = - JSONBoolGenFunc(lengthGen, mapping) - - override def withValueRange(min: Any, max: Any): GeneratorFunction = - throw new IllegalArgumentException("value ranges are not supported for JSON") -} - -class JSONBoolGen(conf: ColumnConf, - defaultValueRange: Option[(Any, Any)] = None) - extends SubstringDataGen(conf, defaultValueRange) { - - override protected def getValGen: GeneratorFunction = JSONBoolGenFunc() - - override def children: Seq[(String, SubstringDataGen)] = Seq.empty -} - -case class JSONNullGenFunc(nullAsString: Boolean, - lengthGen: LengthGeneratorFunction = null, - mapping: LocationToSeedMapping = null) extends GeneratorFunction { - - override def apply(rowLoc: RowLocation): Any = - if (nullAsString) { - UTF8String.fromString("null") +object JSONType { + def selectType(depth: Int, + maxDepth: Int, + r: Random): JSONType = { + val toSelectFrom = if (depth < maxDepth) { + Seq(QuotedJSONString, JSONLong, JSONDouble, JSONArray, JSONObject) } else { - null - } - - - override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONNullGenFunc = - JSONNullGenFunc(nullAsString, lengthGen, mapping) - - override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONNullGenFunc = - JSONNullGenFunc(nullAsString, lengthGen, mapping) - - override def withValueRange(min: Any, max: Any): GeneratorFunction = - throw new IllegalArgumentException("value ranges are not supported for JSON") -} - -class JSONNullGen(nullAsString: Boolean, - conf: ColumnConf, - defaultValueRange: Option[(Any, Any)] = None) - extends SubstringDataGen(conf, defaultValueRange) { - - override protected def getValGen: GeneratorFunction = JSONNullGenFunc(nullAsString) - - override def children: Seq[(String, SubstringDataGen)] = Seq.empty -} - -case class JSONErrorGenFunc(lengthGen: LengthGeneratorFunction = null, - mapping: LocationToSeedMapping = null) extends GeneratorFunction { - - override def apply(rowLoc: RowLocation): Any = { - val len = lengthGen(rowLoc) - val r = DataGen.getRandomFor(rowLoc, mapping) - val buffer = new Array[Byte](len) - var at = 0 - while (at < len) { - // Value range is 32 (Space) to 126 (~) - // But it is almost impossible to show up as valid JSON - buffer(at) = (r.nextInt(126 - 31) + 32).toByte - at += 1 - } - UTF8String.fromBytes(buffer, 0, len) - } - - override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONErrorGenFunc = - JSONErrorGenFunc(lengthGen, mapping) - - override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONErrorGenFunc = - JSONErrorGenFunc(lengthGen, mapping) - - override def withValueRange(min: Any, max: Any): GeneratorFunction = - throw new IllegalArgumentException("value ranges are not supported for JSON") -} - -class JSONErrorGen(conf: ColumnConf, - defaultValueRange: Option[(Any, Any)] = None) - extends SubstringDataGen(conf, defaultValueRange) { - - override protected def getValGen: GeneratorFunction = JSONErrorGenFunc() - - override def children: Seq[(String, SubstringDataGen)] = Seq.empty -} - -case class JSONArrayGenFunc(child: GeneratorFunction, - lengthGen: LengthGeneratorFunction = null, - mapping: LocationToSeedMapping = null) extends GeneratorFunction { - - override def apply(rowLoc: RowLocation): Any = { - val len = lengthGen(rowLoc) - val data = new Array[String](len) - val childRowLoc = rowLoc.withNewChild() - var i = 0 - while (i < len) { - childRowLoc.setLastChildIndex(i) - val v = child(childRowLoc) - if (v == null) { - // A null in an array must look like "null" - data(i) = "null" - } else { - data(i) = v.toString + Seq(QuotedJSONString, JSONLong, JSONDouble) + } + val index = r.nextInt(toSelectFrom.length) + toSelectFrom(index) + } +} + +object QuotedJSONString extends JSONType { + override def appendRandomValue(sb: StringBuilder, + index: Int, + maxStringLength: Int, + maxArrayLength: Int, + maxObjectLength: Int, + depth: Int, + maxDepth: Int, + r: Random): Unit = { + val strValue = r.nextString(r.nextInt(maxStringLength + 1)) + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\b", "\\b") + .replace("\f", "\\f") + sb.append('"') + sb.append(strValue) + sb.append('"') + } +} + +object JSONLong extends JSONType { + override def appendRandomValue(sb: StringBuilder, + index: Int, + maxStringLength: Int, + maxArrayLength: Int, + maxObjectLength: Int, + depth: Int, + maxDepth: Int, + r: Random): Unit = { + sb.append(r.nextLong()) + } +} + +object JSONDouble extends JSONType { + override def appendRandomValue(sb: StringBuilder, + index: Int, + maxStringLength: Int, + maxArrayLength: Int, + maxObjectLength: Int, + depth: Int, + maxDepth: Int, + r: Random): Unit = { + sb.append(r.nextDouble() * 4096.0) + } +} + +object JSONArray extends JSONType { + override def appendRandomValue(sb: StringBuilder, + index: Int, + maxStringLength: Int, + maxArrayLength: Int, + maxObjectLength: Int, + depth: Int, + maxDepth: Int, + r: Random): Unit = { + val childType = JSONType.selectType(depth, maxDepth, r) + val length = r.nextInt(maxArrayLength + 1) + sb.append("[") + (0 until length).foreach { i => + if (i > 0) { + sb.append(",") } - i += 1 + childType.appendRandomValue(sb, i, maxStringLength, maxArrayLength, maxObjectLength, + depth + 1, maxDepth, r) } - val ret = data.mkString("[", ",", "]") - UTF8String.fromString(ret) + sb.append("]") } - - override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONArrayGenFunc = - JSONArrayGenFunc(child, lengthGen, mapping) - - override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONArrayGenFunc = - JSONArrayGenFunc(child, lengthGen, mapping) - - override def withValueRange(min: Any, max: Any): GeneratorFunction = - throw new IllegalArgumentException("value ranges are not supported for JSON") } -class JSONArrayGen(child: SubstringDataGen, - conf: ColumnConf, - defaultValueRange: Option[(Any, Any)] = None) - extends SubstringDataGen(conf, defaultValueRange) { - - override def setCorrelatedKeyGroup(keyGroup: Long, - minSeed: Long, maxSeed: Long, - seedMapping: LocationToSeedMapping): SubstringDataGen = { - super.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) - child.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) - this - } - - override protected def getValGen: GeneratorFunction = JSONArrayGenFunc(child.getGen) - - override def get(name: String): Option[SubstringDataGen] = { - if ("data".equalsIgnoreCase(name) || "child".equalsIgnoreCase(name)) { - Some(child) - } else { - None - } - } - - override def children: Seq[(String, SubstringDataGen)] = Seq(("data", child)) -} - -case class JSONObjectGenFunc(childGens: Array[(String, GeneratorFunction)], - lengthGen: LengthGeneratorFunction = null, - mapping: LocationToSeedMapping = null) extends GeneratorFunction { - override def apply(rowLoc: RowLocation): Any = { - // TODO randomize the order of the children??? - // TODO duplicate child values??? - // The row location does not change for a struct/object - val data = childGens.map { - case (k, gen) => - val key = k.replace("\\", "\\\\") - .replace("\"", "\\\"") - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\b", "\\b") - .replace("\f", "\\f") - val v = gen.apply(rowLoc) - if (v == null) { - "" - } else { - '"' + key + "\":" + v - } +object JSONObject extends JSONType { + override def appendRandomValue(sb: StringBuilder, + index: Int, + maxStringLength: Int, + maxArrayLength: Int, + maxObjectLength: Int, + depth: Int, + maxDepth: Int, + r: Random): Unit = { + val length = r.nextInt(maxObjectLength) + 1 + sb.append("{") + (0 until length).foreach { i => + if (i > 0) { + sb.append(",") + } + sb.append("\"key_") + sb.append(i) + sb.append("_") + sb.append(depth ) + sb.append("\":") + val childType = JSONType.selectType(depth, maxDepth, r) + childType.appendRandomValue(sb, i, maxStringLength, maxArrayLength, maxObjectLength, + depth + 1, maxDepth, r) } - val ret = data.filterNot(_.isEmpty).mkString("{",",","}") - UTF8String.fromString(ret) + sb.append("}") } - - override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONObjectGenFunc = - JSONObjectGenFunc(childGens, lengthGen, mapping) - - override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONObjectGenFunc = - JSONObjectGenFunc(childGens, lengthGen, mapping) - - override def withValueRange(min: Any, max: Any): GeneratorFunction = - throw new IllegalArgumentException("value ranges are not supported for JSON") } -class JSONObjectGen(val children: Seq[(String, SubstringDataGen)], - conf: ColumnConf, - defaultValueRange: Option[(Any, Any)] = None) - extends SubstringDataGen(conf, defaultValueRange) { - - override def setCorrelatedKeyGroup(keyGroup: Long, - minSeed: Long, maxSeed: Long, - seedMapping: LocationToSeedMapping): SubstringDataGen = { - super.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) - children.foreach { - case (_, gen) => - gen.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) - } - this - } - - override def get(name: String): Option[SubstringDataGen] = - children.collectFirst { - case (childName, dataGen) if childName.equalsIgnoreCase(name) => dataGen - } - - override protected def getValGen: GeneratorFunction = { - val childGens = children.map(c => (c._1, c._2.getGen)).toArray - JSONObjectGenFunc(childGens) - } -} +case class JSONGenFunc( + maxStringLength: Int, + maxArrayLength: Int, + maxObjectLength: Int, + maxDepth: Int, + lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { -case class JSONChoiceGenFunc(choices: List[(Double, GeneratorFunction)], - lengthGen: LengthGeneratorFunction = null, - mapping: LocationToSeedMapping = null) extends GeneratorFunction { override def apply(rowLoc: RowLocation): Any = { val r = DataGen.getRandomFor(rowLoc, mapping) - val l = r.nextDouble() - var index = 0 - while (choices(index)._1 < l) { - index += 1 - } - val childRowLoc = rowLoc.withNewChild() - choices(index)._2(childRowLoc) + val sb = new StringBuilder() + JSONObject.appendRandomValue(sb, 0, maxStringLength, maxArrayLength, maxObjectLength, + 0, maxDepth, r) + // For now I am going to have some hard coded keys + UTF8String.fromString(sb.toString()) } - override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONChoiceGenFunc = - JSONChoiceGenFunc(choices, lengthGen, mapping) + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): GeneratorFunction = + JSONGenFunc(maxStringLength, maxArrayLength, maxObjectLength, maxDepth, lengthGen, mapping) - override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONChoiceGenFunc = - JSONChoiceGenFunc(choices, lengthGen, mapping) + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): GeneratorFunction = + JSONGenFunc(maxStringLength, maxArrayLength, maxObjectLength, maxDepth, lengthGen, mapping) override def withValueRange(min: Any, max: Any): GeneratorFunction = - throw new IllegalArgumentException("value ranges are not supported for JSON") -} - -class JSONChoiceGen(val choices: Seq[(Double, String, SubstringDataGen)], - conf: ColumnConf, - defaultValueRange: Option[(Any, Any)] = None) - extends SubstringDataGen(conf, defaultValueRange) { - - override val children: Seq[(String, SubstringDataGen)] = - choices.map { case (_, name, gen) => (name, gen) } - - override def setCorrelatedKeyGroup(keyGroup: Long, - minSeed: Long, maxSeed: Long, - seedMapping: LocationToSeedMapping): SubstringDataGen = { - super.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) - children.foreach { - case (_, gen) => - gen.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) - } - this - } - - override def get(name: String): Option[SubstringDataGen] = - children.collectFirst { - case (childName, dataGen) if childName.equalsIgnoreCase(name) => dataGen - } - - override protected def getValGen: GeneratorFunction = { - val childGens = choices.map(c => (c._1, c._3.getGen)).toList - JSONChoiceGenFunc(childGens) - } + throw new IllegalArgumentException("value ranges are not supported for strings") } case class ASCIIGenFunc( @@ -2451,46 +1672,14 @@ case class ASCIIGenFunc( throw new IllegalArgumentException("value ranges are not supported for strings") } -/** - * This is here to wrap the substring gen function so that its length/settings - * are the ones used when generating a string, and not what was set for the string. - */ -case class SubstringGenFunc( - substringGen: GeneratorFunction, - lengthGen: LengthGeneratorFunction = null, - mapping: LocationToSeedMapping = null) extends GeneratorFunction { - - override def apply(rowLoc: RowLocation): Any = { - substringGen(rowLoc) - } - - // The length and location seed mapping are just ignored for this... - override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): GeneratorFunction = - this - - override def withLocationToSeedMapping(mapping: LocationToSeedMapping): GeneratorFunction = - this - - override def withValueRange(min: Any, max: Any): GeneratorFunction = - throw new IllegalArgumentException("value ranges are not supported for strings") -} - -class StringGen(conf: ColumnConf, - defaultValueRange: Option[(Any, Any)], - var substringDataGen: Option[SubstringDataGen] = None) - extends DataGen(conf, defaultValueRange) { +class StringGen(conf: ColumnConf, defaultValueRange: Option[(Any, Any)]) + extends DataGen(conf, defaultValueRange) { override def dataType: DataType = StringType - override protected def getValGen: GeneratorFunction = - substringDataGen.map(s => SubstringGenFunc(s.getGen)).getOrElse(ASCIIGenFunc()) + override protected def getValGen: GeneratorFunction = ASCIIGenFunc() override def children: Seq[(String, DataGen)] = Seq.empty - - override def setSubstringGen(subgen: Option[SubstringDataGen]): Unit = - substringDataGen = subgen - - override def getSubstringGen: Option[SubstringDataGen] = substringDataGen } case class StructGenFunc(childGens: Array[GeneratorFunction]) extends GeneratorFunction { @@ -2665,6 +1854,7 @@ class MapGen(key: DataGen, override def children: Seq[(String, DataGen)] = Seq(("key", key), ("value", value)) } + object ColumnGen { private def genInternal(rowNumber: Column, dataType: DataType, @@ -2679,8 +1869,8 @@ object ColumnGen { */ class ColumnGen(val dataGen: DataGen) { def setCorrelatedKeyGroup(kg: Long, - minSeed: Long, maxSeed: Long, - seedMapping: LocationToSeedMapping): ColumnGen = { + minSeed: Long, maxSeed: Long, + seedMapping: LocationToSeedMapping): ColumnGen = { dataGen.setCorrelatedKeyGroup(kg, minSeed, maxSeed, seedMapping) this } @@ -2740,11 +1930,6 @@ class ColumnGen(val dataGen: DataGen) { this } - def setGaussianLength(mean: Double, stdDev: Double): ColumnGen = { - dataGen.setGaussianLength(mean, stdDev) - this - } - final def apply(name: String): DataGen = { get(name).getOrElse { throw new IllegalArgumentException(s"$name not a child of $this") @@ -2756,16 +1941,8 @@ class ColumnGen(val dataGen: DataGen) { def gen(rowNumber: Column): Column = { ColumnGen.genInternal(rowNumber, dataGen.dataType, dataGen.nullable, dataGen.getGen) } - - def getSubstring: Option[SubstringDataGen] = dataGen.getSubstringGen - - def substringGen: SubstringDataGen = dataGen.substringGen - - def setSubstringGen(f : ColumnConf => SubstringDataGen): Unit = - dataGen.setSubstringGen(f) } - sealed trait KeyGroupType /** @@ -3015,7 +2192,7 @@ object DBGen { numRows: Long, mapping: OrderedTypeMapping): Seq[(String, ColumnGen)] = { // a bit of a hack with the column num so that we update it before each time... - var conf = ColumnConf(ColumnLocation(tableId, -1, 0), true, numRows) + var conf = ColumnConf(ColumnLocation(tableId, -1), true, numRows) st.toArray.map { sf => if (!mapping.canMap(sf.dataType, mapping)) { throw new IllegalArgumentException(s"$sf is not supported at this time") From 531a9f5e6501baeda67bd12fa223726d5e8c3572 Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Thu, 13 Jun 2024 14:22:50 -0500 Subject: [PATCH 32/79] Add in the ability to fingerprint JSON columns [databricks] (#11060) Also fixed issue with databricks dependency not being what we said it was. Signed-off-by: Robert (Bobby) Evans --- .../spark/sql/tests/datagen/bigDataGen.scala | 1181 ++++++++++++++--- jenkins/databricks/install_deps.py | 4 +- scala2.13/shim-deps/databricks/pom.xml | 8 +- shim-deps/databricks/pom.xml | 8 +- 4 files changed, 1019 insertions(+), 182 deletions(-) diff --git a/datagen/src/main/scala/org/apache/spark/sql/tests/datagen/bigDataGen.scala b/datagen/src/main/scala/org/apache/spark/sql/tests/datagen/bigDataGen.scala index 91335afe4e6..14e0d4e0970 100644 --- a/datagen/src/main/scala/org/apache/spark/sql/tests/datagen/bigDataGen.scala +++ b/datagen/src/main/scala/org/apache/spark/sql/tests/datagen/bigDataGen.scala @@ -16,21 +16,22 @@ package org.apache.spark.sql.tests.datagen +import com.fasterxml.jackson.core.{JsonFactoryBuilder, JsonParser, JsonToken} +import com.fasterxml.jackson.core.json.JsonReadFeature import java.math.{BigDecimal => JavaBigDecimal} import java.sql.{Date, Timestamp} import java.time.{Duration, Instant, LocalDate, LocalDateTime} import java.util - import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.math.BigDecimal.RoundingMode import scala.util.Random -import org.apache.spark.sql.{Column, DataFrame, SparkSession} +import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Expression, XXH64} import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils} -import org.apache.spark.sql.functions.col +import org.apache.spark.sql.functions.{approx_count_distinct, avg, coalesce, col, count, lit, stddev, struct, transform, udf, when} import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.random.XORShiftRandom @@ -79,22 +80,28 @@ class RowLocation(val rowNum: Long, val subRows: Array[Int] = null) { * hash. This makes the generated data correlated for all column/child columns. * @param tableNum a unique ID for the table this is a part of. * @param columnNum the location of the column in the data being generated + * @param substringNum the location of the substring column * @param correlatedKeyGroup the correlated key group this column is a part of, if any. */ -case class ColumnLocation(tableNum: Int, columnNum: Int, correlatedKeyGroup: Option[Long] = None) { - def forNextColumn(): ColumnLocation = ColumnLocation(tableNum, columnNum + 1) +case class ColumnLocation(tableNum: Int, + columnNum: Int, + substringNum: Int, + correlatedKeyGroup: Option[Long] = None) { + def forNextColumn(): ColumnLocation = ColumnLocation(tableNum, columnNum + 1, 0) + def forNextSubstring: ColumnLocation = ColumnLocation(tableNum, columnNum, substringNum + 1) /** * Create a new ColumnLocation that is specifically for a given key group */ def forCorrelatedKeyGroup(keyGroup: Long): ColumnLocation = - ColumnLocation(tableNum, columnNum, Some(keyGroup)) + ColumnLocation(tableNum, columnNum, substringNum, Some(keyGroup)) /** * Hash the location into a single long value. */ - lazy val hashLoc: Long = XXH64.hashLong(tableNum, correlatedKeyGroup.getOrElse(columnNum)) + lazy val hashLoc: Long = XXH64.hashLong(tableNum, + correlatedKeyGroup.getOrElse(XXH64.hashLong(columnNum, substringNum))) } /** @@ -115,6 +122,9 @@ case class ColumnConf(columnLoc: ColumnLocation, def forNextColumn(nullable: Boolean): ColumnConf = ColumnConf(columnLoc.forNextColumn(), nullable, numTableRows) + def forNextSubstring: ColumnConf = + ColumnConf(columnLoc.forNextSubstring, nullable = true, numTableRows) + /** * Create a new configuration based on this, but for a given correlated key group. */ @@ -303,6 +313,23 @@ case class VarLengthGeneratorFunction(minLength: Int, maxLength: Int) extends } } +case class StdDevLengthGen(mean: Double, + stdDev: Double, + mapping: LocationToSeedMapping = null) extends + LengthGeneratorFunction { + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): LengthGeneratorFunction = + StdDevLengthGen(mean, stdDev, mapping) + + override def apply(rowLoc: RowLocation): Int = { + val r = DataGen.getRandomFor(rowLoc, mapping) + val g = r.nextGaussian() // g has a mean of 0 and a stddev of 1.0 + val adjusted = mean + (g * stdDev) + // If the range of seed is too small compared to the stddev and mean we will + // end up with an invalid distribution, but they asked for it. + math.max(0, math.round(adjusted).toInt) + } +} + /** * Generate nulls with a given probability. * @param prob 0.0 to 1.0 for how often nulls should appear in the output. @@ -562,11 +589,8 @@ case class DataGenExpr(child: Expression, } } -/** - * Base class for generating a column/sub-column. This holds configuration for the column, - * and handles what is needed to convert it into GeneratorFunction - */ -abstract class DataGen(var conf: ColumnConf, +abstract class CommonDataGen( + var conf: ColumnConf, defaultValueRange: Option[(Any, Any)], var seedMapping: LocationToSeedMapping = FlatDistribution(), var nullMapping: LocationToSeedMapping = FlatDistribution(), @@ -576,26 +600,25 @@ abstract class DataGen(var conf: ColumnConf, protected var valueRange: Option[(Any, Any)] = defaultValueRange /** - * Set a value range for this data gen. + * Set a value range */ - def setValueRange(min: Any, max: Any): DataGen = { + def setValueRange(min: Any, max: Any): CommonDataGen = { valueRange = Some((min, max)) this } /** - * Set a custom GeneratorFunction to use for this column. + * Set a custom GeneratorFunction */ - def setValueGen(f: GeneratorFunction): DataGen = { + def setValueGen(f: GeneratorFunction): CommonDataGen = { userProvidedValueGen = Some(f) this } /** - * Set a NullGeneratorFunction for this column. This will not be used - * if the column is not nullable. + * Set a NullGeneratorFunction */ - def setNullGen(f: NullGeneratorFunction): DataGen = { + def setNullGen(f: NullGeneratorFunction): CommonDataGen = { this.userProvidedNullGen = Some(f) this } @@ -604,12 +627,12 @@ abstract class DataGen(var conf: ColumnConf, * Set the probability of a null appearing in the output. The probability should be * 0.0 to 1.0. */ - def setNullProbability(probability: Double): DataGen = { + def setNullProbability(probability: Double): CommonDataGen = { this.userProvidedNullGen = Some(NullProbabilityGenerationFunction(probability)) this } - def setNullProbabilityRecursively(probability: Double): DataGen = { + def setNullProbabilityRecursively(probability: Double): CommonDataGen = { this.userProvidedNullGen = Some(NullProbabilityGenerationFunction(probability)) children.foreach { case (_, dataGen) => @@ -621,7 +644,7 @@ abstract class DataGen(var conf: ColumnConf, /** * Set a specific location to seed mapping for the value generation. */ - def setSeedMapping(seedMapping: LocationToSeedMapping): DataGen = { + def setSeedMapping(seedMapping: LocationToSeedMapping): CommonDataGen = { this.seedMapping = seedMapping this } @@ -629,7 +652,7 @@ abstract class DataGen(var conf: ColumnConf, /** * Set a specific location to seed mapping for the null generation. */ - def setNullMapping(nullMapping: LocationToSeedMapping): DataGen = { + def setNullMapping(nullMapping: LocationToSeedMapping): CommonDataGen = { this.nullMapping = nullMapping this } @@ -638,7 +661,7 @@ abstract class DataGen(var conf: ColumnConf, * Set a specific LengthGeneratorFunction to use. This will only be used if * the datatype needs a length. */ - def setLengthGen(lengthGen: LengthGeneratorFunction): DataGen = { + def setLengthGen(lengthGen: LengthGeneratorFunction): CommonDataGen = { this.lengthGen = lengthGen this } @@ -646,25 +669,30 @@ abstract class DataGen(var conf: ColumnConf, /** * Set the length generation to be a fixed length. */ - def setLength(len: Int): DataGen = { + def setLength(len: Int): CommonDataGen = { this.lengthGen = FixedLengthGeneratorFunction(len) this } - def setLength(minLen: Int, maxLen: Int) = { + def setLength(minLen: Int, maxLen: Int): CommonDataGen = { this.lengthGen = VarLengthGeneratorFunction(minLen, maxLen) this } + def setGaussianLength(mean: Double, stdDev: Double): CommonDataGen = { + this.lengthGen = StdDevLengthGen(mean, stdDev) + this + } + /** * Add this column to a specific correlated key group. This should not be * called directly by users. */ def setCorrelatedKeyGroup(keyGroup: Long, - minSeed: Long, maxSeed: Long, - seedMapping: LocationToSeedMapping): DataGen = { + minSeed: Long, maxSeed: Long, + seedMapping: LocationToSeedMapping): CommonDataGen = { conf = conf.forCorrelatedKeyGroup(keyGroup) - .forSeedRange(minSeed, maxSeed) + .forSeedRange(minSeed, maxSeed) this.seedMapping = seedMapping this } @@ -672,7 +700,7 @@ abstract class DataGen(var conf: ColumnConf, /** * Set a range of seed values that should be returned by the LocationToSeedMapping */ - def setSeedRange(min: Long, max: Long): DataGen = { + def setSeedRange(min: Long, max: Long): CommonDataGen = { conf = conf.forSeedRange(min, max) this } @@ -681,7 +709,7 @@ abstract class DataGen(var conf: ColumnConf, * Get the default value generator for this specific data gen. */ protected def getValGen: GeneratorFunction - def children: Seq[(String, DataGen)] + def children: Seq[(String, CommonDataGen)] /** * Get the final ready to use GeneratorFunction for the data generator. @@ -690,8 +718,8 @@ abstract class DataGen(var conf: ColumnConf, val sm = seedMapping.withColumnConf(conf) val lg = lengthGen.withLocationToSeedMapping(sm) var valGen = userProvidedValueGen.getOrElse(getValGen) - .withLocationToSeedMapping(sm) - .withLengthGeneratorFunction(lg) + .withLocationToSeedMapping(sm) + .withLengthGeneratorFunction(lg) valueRange.foreach { case (min, max) => valGen = valGen.withValueRange(min, max) @@ -700,35 +728,75 @@ abstract class DataGen(var conf: ColumnConf, val nullColConf = conf.forNulls val nm = nullMapping.withColumnConf(nullColConf) userProvidedNullGen.get - .withWrapped(valGen) - .withLocationToSeedMapping(nm) + .withWrapped(valGen) + .withLocationToSeedMapping(nm) } else { valGen } } - /** - * Get the data type for this column - */ - def dataType: DataType - /** * Is this column nullable or not. */ def nullable: Boolean = conf.nullable /** - * Get a child column for a given name, if it has one. + * Get a child for a given name, if it has one. */ - final def apply(name: String): DataGen = { + final def apply(name: String): CommonDataGen = { get(name).getOrElse{ throw new IllegalStateException(s"Could not find a child $name for $this") } } - def get(name: String): Option[DataGen] = None + def get(name: String): Option[CommonDataGen] = None +} + + +/** + * Base class for generating a column/sub-column. This holds configuration + * for the column, and handles what is needed to convert it into GeneratorFunction + */ +abstract class DataGen( + conf: ColumnConf, + defaultValueRange: Option[(Any, Any)], + seedMapping: LocationToSeedMapping = FlatDistribution(), + nullMapping: LocationToSeedMapping = FlatDistribution(), + lengthGen: LengthGeneratorFunction = FixedLengthGeneratorFunction(10)) extends + CommonDataGen(conf, defaultValueRange, seedMapping, nullMapping, lengthGen) { + + /** + * Get the data type for this column + */ + def dataType: DataType + + override def get(name: String): Option[DataGen] = None + + def getSubstringGen: Option[SubstringDataGen] = None + + def substringGen: SubstringDataGen = + getSubstringGen.getOrElse( + throw new IllegalArgumentException("substring data gen was not set")) + + def setSubstringGen(f : ColumnConf => SubstringDataGen): Unit = + setSubstringGen(Option(f(conf.forNextSubstring))) + + def setSubstringGen(subgen: Option[SubstringDataGen]): Unit = + throw new IllegalArgumentException("substring data gens can only be set for a STRING") } +/** + * Base class for generating a sub-string. This holds configuration + * for the substring, and handles what is needed to convert it into a GeneratorFunction + */ +abstract class SubstringDataGen( + conf: ColumnConf, + defaultValueRange: Option[(Any, Any)], + seedMapping: LocationToSeedMapping = FlatDistribution(), + nullMapping: LocationToSeedMapping = FlatDistribution(), + lengthGen: LengthGeneratorFunction = FixedLengthGeneratorFunction(10)) extends + CommonDataGen(conf, defaultValueRange, seedMapping, nullMapping, lengthGen) {} + /** * A special GeneratorFunction that just returns the computed seed. This is helpful for * debugging distributions or if you want long values without any abstraction in between. @@ -1494,155 +1562,866 @@ class FloatGen(conf: ColumnConf, defaultValueRange: Option[(Any, Any)]) override def children: Seq[(String, DataGen)] = Seq.empty } -trait JSONType { - def appendRandomValue(sb: StringBuilder, - index: Int, - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - depth: Int, - maxDepth: Int, - r: Random): Unit -} +case class JsonPathElement(name: String, is_array: Boolean) +case class JsonLevel(path: Array[JsonPathElement], data_type: String, length: Int, value: String) {} + +object JsonColumnStats { + private def printHelp(): Unit = { + println("JSON Fingerprinting Tool:") + println("PARAMS: ") + println(" is a path to a Spark dataframe to read in") + println(" is a path in a Spark file system to write out fingerprint data to.") + println() + println("OPTIONS:") + println(" --json= where is the name of a top level String column") + println(" --anon= where is a SEED used to anonymize the JSON keys ") + println(" and column names.") + println(" --input_format= where is parquet or ORC. Defaults to parquet.") + println(" --overwrite to enable overwriting the fingerprint output.") + println(" --debug to enable some debug information to be printed out") + println(" --help to print out this help message") + println() + } + + def main(args: Array[String]): Unit = { + var inputPath = Option.empty[String] + var outputPath = Option.empty[String] + val jsonColumns = ArrayBuffer.empty[String] + var anonSeed = Option.empty[Long] + var debug = false + var argsDone = false + var format = "parquet" + var overwrite = false + + args.foreach { + case a if !argsDone && a.startsWith("--json=") => + jsonColumns += a.substring("--json=".length) + case a if !argsDone && a.startsWith("--anon=") => + anonSeed = Some(a.substring("--anon=".length).toLong) + case a if !argsDone && a.startsWith("--input_format=") => + format = a.substring("--input_format=".length).toLowerCase(java.util.Locale.US) + case "--overwrite" if !argsDone => + overwrite = true + case "--debug" if !argsDone => + debug = true + case "--help" if !argsDone => + printHelp() + System.exit(0) + case "--" if !argsDone => + argsDone = true + case a if !argsDone && a.startsWith("--") => // "--" was covered above already + println(s"ERROR $a is not a supported argument") + printHelp() + System.exit(-1) + case a if inputPath.isEmpty => + inputPath = Some(a) + case a if outputPath.isEmpty => + outputPath = Some(a) + case a => + println(s"ERROR only two arguments are supported. Found $a") + printHelp() + System.exit(-1) + } + if (outputPath.isEmpty) { + println("ERROR both an inputPath and an outputPath are required") + printHelp() + System.exit(-1) + } + + val spark = SparkSession.builder.getOrCreate() + spark.sparkContext.setLogLevel("WARN") + + val df = spark.read.format(format).load(inputPath.get) + jsonColumns.foreach { column => + val fp = fingerPrint(df, df(column), anonSeed) + val name = anonSeed.map(s => anonymizeString(column, s)).getOrElse(column) + val fullOutPath = s"${outputPath.get}/$name" + var writer = fp.write + if (overwrite) { + writer = writer.mode("overwrite") + } + if (debug) { + anonSeed.foreach { s => + println(s"Keys and columns will be anonymized with seed $s") + } + println(s"Writing $column fingerprint to $fullOutPath") + spark.time(writer.parquet(fullOutPath)) + println(s"Wrote ${spark.read.parquet(fullOutPath).count} rows") + spark.read.parquet(fullOutPath).show() + } else { + writer.parquet(fullOutPath) + } + } + } -object JSONType { - def selectType(depth: Int, - maxDepth: Int, - r: Random): JSONType = { - val toSelectFrom = if (depth < maxDepth) { - Seq(QuotedJSONString, JSONLong, JSONDouble, JSONArray, JSONObject) - } else { - Seq(QuotedJSONString, JSONLong, JSONDouble) - } - val index = r.nextInt(toSelectFrom.length) - toSelectFrom(index) - } -} - -object QuotedJSONString extends JSONType { - override def appendRandomValue(sb: StringBuilder, - index: Int, - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - depth: Int, - maxDepth: Int, - r: Random): Unit = { - val strValue = r.nextString(r.nextInt(maxStringLength + 1)) - .replace("\\", "\\\\") - .replace("\"", "\\\"") - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\b", "\\b") - .replace("\f", "\\f") - sb.append('"') - sb.append(strValue) - sb.append('"') - } -} - -object JSONLong extends JSONType { - override def appendRandomValue(sb: StringBuilder, - index: Int, - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - depth: Int, - maxDepth: Int, - r: Random): Unit = { - sb.append(r.nextLong()) - } -} - -object JSONDouble extends JSONType { - override def appendRandomValue(sb: StringBuilder, - index: Int, - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - depth: Int, - maxDepth: Int, - r: Random): Unit = { - sb.append(r.nextDouble() * 4096.0) - } -} - -object JSONArray extends JSONType { - override def appendRandomValue(sb: StringBuilder, - index: Int, - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - depth: Int, - maxDepth: Int, - r: Random): Unit = { - val childType = JSONType.selectType(depth, maxDepth, r) - val length = r.nextInt(maxArrayLength + 1) - sb.append("[") + case class JsonNodeStats(count: Long, meanLen: Double, stdDevLength: Double, dc: Long) + + class JsonNode() { + private val forDataType = + mutable.HashMap[String, (JsonNodeStats, mutable.HashMap[String, JsonNode])]() + + def getChild(name: String, isArray: Boolean): JsonNode = { + val dt = if (isArray) { "ARRAY" } else { "OBJECT" } + val typed = forDataType.getOrElse(dt, + throw new IllegalArgumentException(s"$dt is not a set data type yet.")) + typed._2.getOrElse(name, + throw new IllegalArgumentException(s"$name is not a child when the type is $dt")) + } + + def contains(name: String, isArray: Boolean): Boolean = { + val dt = if (isArray) { "ARRAY" } else { "OBJECT" } + forDataType.get(dt).exists { children => + children._2.contains(name) + } + } + + def addChild(name: String, isArray: Boolean): JsonNode = { + val dt = if (isArray) { "ARRAY" } else { "OBJECT" } + val found = forDataType.getOrElse(dt, + throw new IllegalArgumentException(s"$dt was not already added as a data type")) + if (found._2.contains(name)) { + throw new IllegalArgumentException(s"$dt already has a child named $name") + } + val node = new JsonNode() + found._2.put(name, node) + node + } + + def addChoice(dt: String, stats: JsonNodeStats): Unit = { + if (forDataType.contains(dt)) { + throw new IllegalArgumentException(s"$dt was already added as a data type") + } + forDataType.put(dt, (stats, new mutable.HashMap[String, JsonNode]())) + } + + override def toString: String = { + forDataType.toString() + } + + def totalCount: Long = { + forDataType.values.map{ case (stats, _) => stats.count}.sum + } + + private def makeNoChoiceGenRecursive(dt: String, + children: mutable.HashMap[String, JsonNode], + cc: ColumnConf): (SubstringDataGen, ColumnConf) = { + var c = cc + val ret = dt match { + case "LONG" => new JSONLongGen(c) + case "DOUBLE" => new JSONDoubleGen(c) + case "BOOLEAN" => new JSONBoolGen(c) + case "NULL" => new JSONNullGen(false, c) + case "VALUE_NULL" => new JSONNullGen(true, c) + case "ERROR" => new JSONErrorGen(c) + case "STRING" => new JSONStringGen(c) + case "ARRAY" => + val child = if (children.isEmpty) { + // A corner case, we will just make it a BOOL column and it will be ignored + val tmp = new JSONBoolGen(c) + c = c.forNextSubstring + tmp + } else { + val tmp = children.values.head.makeGenRecursive(c) + c = tmp._2 + tmp._1 + } + new JSONArrayGen(child, c) + case "OBJECT" => + val childGens = if (children.isEmpty) { + Seq.empty + } else { + children.toSeq.map { + case (k, node) => + val tmp = node.makeGenRecursive(c) + c = tmp._2 + (k, tmp._1) + } + } + new JSONObjectGen(childGens, c) + case other => + throw new IllegalArgumentException(s"$other is not a leaf node type") + } + (ret, c.forNextSubstring) + } + + private def makeGenRecursive(cc: ColumnConf): (SubstringDataGen, ColumnConf) = { + var c = cc + // We are going to recursively walk the tree for all of the values. + if (forDataType.size == 1) { + // We don't need a choice at all. This makes it simpler.. + val (dt, (_, children)) = forDataType.head + makeNoChoiceGenRecursive(dt, children, c) + } else { + val totalSum = forDataType.map(f => f._2._1.count).sum.toDouble + var runningSum = 0L + val allChoices = ArrayBuffer[(Double, String, SubstringDataGen)]() + forDataType.foreach { + case (dt, (stats, children)) => + val tmp = makeNoChoiceGenRecursive(dt, children, c) + c = tmp._2 + runningSum += stats.count + allChoices.append((runningSum/totalSum, dt, tmp._1)) + } + + val ret = new JSONChoiceGen(allChoices.toSeq, c) + (ret, c.forNextSubstring) + } + } + + def makeGen(cc: ColumnConf): SubstringDataGen = { + val (ret, _) = makeGenRecursive(cc) + ret + } + + def setStatsSingle(dg: CommonDataGen, + dt: String, + stats: JsonNodeStats, + nullPct: Double): Unit = { + + val includeLength = dt != "OBJECT" && dt != "BOOLEAN" && dt != "NULL" && dt != "VALUE_NULL" + val includeNullPct = nullPct > 0.0 + if (includeLength) { + dg.setGaussianLength(stats.meanLen, stats.stdDevLength) + } + if (includeNullPct) { + dg.setNullProbability(nullPct) + } + dg.setSeedRange(1, stats.dc) + } + + def setStats(dg: CommonDataGen, + parentCount: Option[Long]): Unit = { + // We are going to recursively walk the tree... + if (forDataType.size == 1) { + // We don't need a choice at all. This makes it simpler.. + val (dt, (stats, children)) = forDataType.head + val nullPct = parentCount.map { pc => + (pc - stats.count).toDouble/pc + }.getOrElse(0.0) + setStatsSingle(dg, dt, stats, nullPct) + val myCount = if (dt == "OBJECT") { + Some(totalCount) + } else { + None + } + children.foreach { + case (name, node) => + node.setStats(dg(name), myCount) + } + } else { + // We have choices to make between different types. + // The null percent cannot be calculated for each individual choice + // but is calculated on the group as a whole instead + parentCount.foreach { pc => + val tc = totalCount + val choiceNullPct = (pc - tc).toDouble / pc + if (choiceNullPct > 0.0) { + dg.setNullProbability(choiceNullPct) + } + } + forDataType.foreach { + case (dt, (stats, children)) => + // When there is a choice the name to access it is the data type + val choiceDg = dg(dt) + setStatsSingle(choiceDg, dt, stats, 0.0) + children.foreach { + case (name, node) => + val myCount = if (dt == "OBJECT") { + // Here we only want the count for the OBJECTs + Some(stats.count) + } else { + None + } + node.setStats(choiceDg(name), myCount) + } + } + } + } + } + + private lazy val jsonFactory = new JsonFactoryBuilder() + // The two options below enabled for Hive compatibility + .enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS) + .enable(JsonReadFeature.ALLOW_SINGLE_QUOTES) + .build() + + private def processNext(parser: JsonParser, + currentPath: ArrayBuffer[JsonPathElement], + output: ArrayBuffer[JsonLevel]): Unit = { + parser.currentToken() match { + case JsonToken.START_OBJECT => + parser.nextToken() + while (parser.currentToken() != JsonToken.END_OBJECT) { + processNext(parser, currentPath, output) + } + output.append(JsonLevel(currentPath.toArray, "OBJECT", 0, "")) + parser.nextToken() + case JsonToken.START_ARRAY => + currentPath.append(JsonPathElement("data", is_array = true)) + parser.nextToken() + var length = 0 + while (parser.currentToken() != JsonToken.END_ARRAY) { + length += 1 + processNext(parser, currentPath, output) + } + currentPath.remove(currentPath.length - 1) + output.append(JsonLevel(currentPath.toArray, "ARRAY", length, "")) + parser.nextToken() + case JsonToken.FIELD_NAME => + currentPath.append(JsonPathElement(parser.getCurrentName, is_array = false)) + parser.nextToken() + processNext(parser, currentPath, output) + currentPath.remove(currentPath.length - 1) + case JsonToken.VALUE_NUMBER_INT => + val length = parser.getValueAsString.getBytes("UTF-8").length + output.append(JsonLevel(currentPath.toArray, "LONG", length, parser.getValueAsString)) + parser.nextToken() + case JsonToken.VALUE_NUMBER_FLOAT => + val length = parser.getValueAsString.getBytes("UTF-8").length + output.append(JsonLevel(currentPath.toArray, "DOUBLE", length, parser.getValueAsString)) + parser.nextToken() + case JsonToken.VALUE_TRUE | JsonToken.VALUE_FALSE => + val length = parser.getValueAsString.getBytes("UTF-8").length + output.append(JsonLevel(currentPath.toArray, "BOOLEAN", length, parser.getValueAsString)) + parser.nextToken() + case JsonToken.VALUE_NULL | null => + output.append(JsonLevel(currentPath.toArray, "VALUE_NULL", 4, "NULL")) + parser.nextToken() + case JsonToken.VALUE_STRING => + val length = parser.getValueAsString.getBytes("UTF-8").length + output.append(JsonLevel(currentPath.toArray, "STRING", length, parser.getValueAsString)) + parser.nextToken() + case other => + throw new IllegalStateException(s"DON'T KNOW HOW TO DEAL WITH $other") + } + } + + def jsonStatsUdf(json: String): Array[JsonLevel] = { + val output = new ArrayBuffer[JsonLevel]() + try { + val currentPath = new ArrayBuffer[JsonPathElement]() + if (json == null) { + output.append(JsonLevel(Array.empty, "NULL", 0, "")) + } else { + val parser = jsonFactory.createParser(json) + try { + parser.nextToken() + processNext(parser, currentPath, output) + } finally { + parser.close() + } + } + } catch { + case _: com.fasterxml.jackson.core.JsonParseException => + output.clear() + output.append(JsonLevel(Array.empty, "ERROR", json.getBytes("UTF-8").length, json)) + } + output.toArray + } + + private lazy val extractPaths = udf(json => jsonStatsUdf(json)) + + def anonymizeString(str: String, seed: Long): String = { + val length = str.length + val data = new Array[Byte](length) + val hash = XXH64.hashLong(str.hashCode, seed) + val r = new Random() + r.setSeed(hash) (0 until length).foreach { i => - if (i > 0) { - sb.append(",") + val tmp = r.nextInt(16) + data(i) = (tmp + 'A').toByte + } + new String(data) + } + + private lazy val anonPath = udf((str, seed) => anonymizeString(str, seed)) + + def anonymizeFingerPrint(df: DataFrame, anonSeed: Long): DataFrame = { + df.withColumn("tmp", transform(col("path"), + o => { + val name = o("name") + val isArray = o("is_array") + val anon = anonPath(name, lit(anonSeed)) + val newName = when(isArray, name).otherwise(anon).alias("name") + struct(newName, isArray) + })) + .drop("path").withColumnRenamed("tmp", "path") + .orderBy("path", "dt") + .selectExpr("path", "dt","c","mean_len","stddev_len","distinct","version") + } + + def fingerPrint(df: DataFrame, column: Column, anonymize: Option[Long] = None): DataFrame = { + val ret = df.select(extractPaths(column).alias("paths")) + .selectExpr("explode_outer(paths) as p") + .selectExpr("p.path as path", "p.data_type as dt", "p.length as len", "p.value as value") + .groupBy(col("path"), col("dt")).agg( + count(lit(1)).alias("c"), + avg(col("len")).alias("mean_len"), + coalesce(stddev(col("len")), lit(0.0)).alias("stddev_len"), + approx_count_distinct(col("value")).alias("distinct")) + .orderBy("path", "dt").withColumn("version", lit("0.1")) + .selectExpr("path", "dt","c","mean_len","stddev_len","distinct","version") + + anonymize.map { anonSeed => + anonymizeFingerPrint(ret, anonSeed) + }.getOrElse(ret) + } + + def apply(aggForColumn: DataFrame, genColumn: ColumnGen): Unit = + apply(aggForColumn, genColumn.dataGen) + + private val expectedSchema = StructType.fromDDL( + "path ARRAY>," + + "dt STRING," + + "c BIGINT," + + "mean_len DOUBLE," + + "stddev_len DOUBLE," + + "distinct BIGINT," + + "version STRING") + + def apply(aggForColumn: DataFrame, gen: DataGen): Unit = { + val aggData = aggForColumn.orderBy("path", "dt").collect() + val rootNode: JsonNode = new JsonNode() + assert(aggData.length > 0) + val schema = aggData.head.schema + assert(schema.length == expectedSchema.length) + schema.fields.zip(expectedSchema.fields).foreach { + case(found, expected) => + assert(found.name == expected.name) + // TODO we can worry about the exact types later if we need to + } + assert(aggData.head.getString(6) == "0.1") + aggData.foreach { row => + val fullPath = row.getAs[mutable.WrappedArray[Row]](0) + val parsedPath = fullPath.map(r => (r.getString(0), r.getBoolean(1))).toList + val dt = row.getString(1) + val count = row.getLong(2) + val meanLen = row.getDouble(3) + val stdLen = row.getDouble(4) + val dc = row.getLong(5) + + val stats = JsonNodeStats(count, meanLen, stdLen, dc) + var currentNode = rootNode + // Find everything up to the last path element + if (parsedPath.length > 1) { + parsedPath.slice(0, parsedPath.length - 1).foreach { + case (name, isArray) => + currentNode = currentNode.getChild(name, isArray) + } + } + + if (parsedPath.nonEmpty) { + // For the last path element (that is not the root element) we might need to add it + // as a child + val (name, isArray) = parsedPath.last + if (!currentNode.contains(name, isArray)) { + currentNode.addChild(name, isArray) + } + currentNode = currentNode.getChild(name, isArray) } - childType.appendRandomValue(sb, i, maxStringLength, maxArrayLength, maxObjectLength, - depth + 1, maxDepth, r) + currentNode.addChoice(dt, stats) } - sb.append("]") + + gen.setSubstringGen(cc => rootNode.makeGen(cc)) + rootNode.setStats(gen.substringGen, None) } } -object JSONObject extends JSONType { - override def appendRandomValue(sb: StringBuilder, - index: Int, - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - depth: Int, - maxDepth: Int, - r: Random): Unit = { - val length = r.nextInt(maxObjectLength) + 1 - sb.append("{") - (0 until length).foreach { i => - if (i > 0) { - sb.append(",") + +case class JSONStringGenFunc(lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = { + val len = lengthGen(rowLoc) + val r = DataGen.getRandomFor(rowLoc, mapping) + val buffer = new Array[Byte](len) + var at = 0 + while (at < len) { + // Value range is 32 (Space) to 126 (~) + buffer(at) = (r.nextInt(126 - 31) + 32).toByte + at += 1 + } + val strVal = new String(buffer, 0, len) + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\b", "\\b") + .replace("\f", "\\f") + '"' + strVal + '"' + } + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONStringGenFunc = + JSONStringGenFunc(lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONStringGenFunc = + JSONStringGenFunc(lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONStringGen(conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override protected def getValGen: GeneratorFunction = JSONStringGenFunc() + + override def children: Seq[(String, SubstringDataGen)] = Seq.empty +} + +case class JSONLongGenFunc(lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = { + val len = math.max(lengthGen(rowLoc), 1) // We need at least 1 long for a valid value + val r = DataGen.getRandomFor(rowLoc, mapping) + val buffer = new Array[Byte](len) + var at = 0 + while (at < len) { + if (at == 0) { + // No leading 0's + buffer(at) = (r.nextInt(9) + '1').toByte + } else { + buffer(at) = (r.nextInt(10) + '0').toByte } - sb.append("\"key_") - sb.append(i) - sb.append("_") - sb.append(depth ) - sb.append("\":") - val childType = JSONType.selectType(depth, maxDepth, r) - childType.appendRandomValue(sb, i, maxStringLength, maxArrayLength, maxObjectLength, - depth + 1, maxDepth, r) + at += 1 } - sb.append("}") + new String(buffer, 0, len) } + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONLongGenFunc = + JSONLongGenFunc(lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONLongGenFunc = + JSONLongGenFunc(lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") } -case class JSONGenFunc( - maxStringLength: Int, - maxArrayLength: Int, - maxObjectLength: Int, - maxDepth: Int, - lengthGen: LengthGeneratorFunction = null, - mapping: LocationToSeedMapping = null) extends GeneratorFunction { +class JSONLongGen(conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override protected def getValGen: GeneratorFunction = JSONLongGenFunc() + + override def children: Seq[(String, SubstringDataGen)] = Seq.empty +} + +case class JSONDoubleGenFunc(lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { override def apply(rowLoc: RowLocation): Any = { + val len = math.max(lengthGen(rowLoc), 3) // We have to have at least 3 chars NUM.NUM val r = DataGen.getRandomFor(rowLoc, mapping) - val sb = new StringBuilder() - JSONObject.appendRandomValue(sb, 0, maxStringLength, maxArrayLength, maxObjectLength, - 0, maxDepth, r) - // For now I am going to have some hard coded keys - UTF8String.fromString(sb.toString()) + val beforeLen = if (len == 3) { 1 } else { r.nextInt(len - 3) + 1 } + val buffer = new Array[Byte](len) + var at = 0 + while (at < len) { + if (at == 0) { + // No leading 0's + buffer(at) = (r.nextInt(9) + '1').toByte + } else if (at == beforeLen) { + buffer(at) = '.' + } else { + buffer(at) = (r.nextInt(10) + '0').toByte + } + at += 1 + } + UTF8String.fromBytes(buffer, 0, len) } - override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): GeneratorFunction = - JSONGenFunc(maxStringLength, maxArrayLength, maxObjectLength, maxDepth, lengthGen, mapping) + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONDoubleGenFunc = + JSONDoubleGenFunc(lengthGen, mapping) - override def withLocationToSeedMapping(mapping: LocationToSeedMapping): GeneratorFunction = - JSONGenFunc(maxStringLength, maxArrayLength, maxObjectLength, maxDepth, lengthGen, mapping) + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONDoubleGenFunc = + JSONDoubleGenFunc(lengthGen, mapping) override def withValueRange(min: Any, max: Any): GeneratorFunction = - throw new IllegalArgumentException("value ranges are not supported for strings") + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONDoubleGen(conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override protected def getValGen: GeneratorFunction = JSONDoubleGenFunc() + + override def children: Seq[(String, SubstringDataGen)] = Seq.empty +} + +case class JSONBoolGenFunc(lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = { + val r = DataGen.getRandomFor(rowLoc, mapping) + val ret = if (r.nextBoolean()) "true" else "false" + UTF8String.fromString(ret) + } + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONBoolGenFunc = + JSONBoolGenFunc(lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONBoolGenFunc = + JSONBoolGenFunc(lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONBoolGen(conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override protected def getValGen: GeneratorFunction = JSONBoolGenFunc() + + override def children: Seq[(String, SubstringDataGen)] = Seq.empty +} + +case class JSONNullGenFunc(nullAsString: Boolean, + lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = + if (nullAsString) { + UTF8String.fromString("null") + } else { + null + } + + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONNullGenFunc = + JSONNullGenFunc(nullAsString, lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONNullGenFunc = + JSONNullGenFunc(nullAsString, lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONNullGen(nullAsString: Boolean, + conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override protected def getValGen: GeneratorFunction = JSONNullGenFunc(nullAsString) + + override def children: Seq[(String, SubstringDataGen)] = Seq.empty +} + +case class JSONErrorGenFunc(lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = { + val len = lengthGen(rowLoc) + val r = DataGen.getRandomFor(rowLoc, mapping) + val buffer = new Array[Byte](len) + var at = 0 + while (at < len) { + // Value range is 32 (Space) to 126 (~) + // But it is almost impossible to show up as valid JSON + buffer(at) = (r.nextInt(126 - 31) + 32).toByte + at += 1 + } + UTF8String.fromBytes(buffer, 0, len) + } + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONErrorGenFunc = + JSONErrorGenFunc(lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONErrorGenFunc = + JSONErrorGenFunc(lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONErrorGen(conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override protected def getValGen: GeneratorFunction = JSONErrorGenFunc() + + override def children: Seq[(String, SubstringDataGen)] = Seq.empty +} + +case class JSONArrayGenFunc(child: GeneratorFunction, + lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = { + val len = lengthGen(rowLoc) + val data = new Array[String](len) + val childRowLoc = rowLoc.withNewChild() + var i = 0 + while (i < len) { + childRowLoc.setLastChildIndex(i) + val v = child(childRowLoc) + if (v == null) { + // A null in an array must look like "null" + data(i) = "null" + } else { + data(i) = v.toString + } + i += 1 + } + val ret = data.mkString("[", ",", "]") + UTF8String.fromString(ret) + } + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONArrayGenFunc = + JSONArrayGenFunc(child, lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONArrayGenFunc = + JSONArrayGenFunc(child, lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONArrayGen(child: SubstringDataGen, + conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override def setCorrelatedKeyGroup(keyGroup: Long, + minSeed: Long, maxSeed: Long, + seedMapping: LocationToSeedMapping): SubstringDataGen = { + super.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) + child.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) + this + } + + override protected def getValGen: GeneratorFunction = JSONArrayGenFunc(child.getGen) + + override def get(name: String): Option[SubstringDataGen] = { + if ("data".equalsIgnoreCase(name) || "child".equalsIgnoreCase(name)) { + Some(child) + } else { + None + } + } + + override def children: Seq[(String, SubstringDataGen)] = Seq(("data", child)) +} + +case class JSONObjectGenFunc(childGens: Array[(String, GeneratorFunction)], + lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + override def apply(rowLoc: RowLocation): Any = { + // TODO randomize the order of the children??? + // TODO duplicate child values??? + // The row location does not change for a struct/object + val data = childGens.map { + case (k, gen) => + val key = k.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\b", "\\b") + .replace("\f", "\\f") + val v = gen.apply(rowLoc) + if (v == null) { + "" + } else { + '"' + key + "\":" + v + } + } + val ret = data.filterNot(_.isEmpty).mkString("{",",","}") + UTF8String.fromString(ret) + } + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONObjectGenFunc = + JSONObjectGenFunc(childGens, lengthGen, mapping) + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONObjectGenFunc = + JSONObjectGenFunc(childGens, lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONObjectGen(val children: Seq[(String, SubstringDataGen)], + conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override def setCorrelatedKeyGroup(keyGroup: Long, + minSeed: Long, maxSeed: Long, + seedMapping: LocationToSeedMapping): SubstringDataGen = { + super.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) + children.foreach { + case (_, gen) => + gen.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) + } + this + } + + override def get(name: String): Option[SubstringDataGen] = + children.collectFirst { + case (childName, dataGen) if childName.equalsIgnoreCase(name) => dataGen + } + + override protected def getValGen: GeneratorFunction = { + val childGens = children.map(c => (c._1, c._2.getGen)).toArray + JSONObjectGenFunc(childGens) + } +} + +case class JSONChoiceGenFunc(choices: List[(Double, GeneratorFunction)], + lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + override def apply(rowLoc: RowLocation): Any = { + val r = DataGen.getRandomFor(rowLoc, mapping) + val l = r.nextDouble() + var index = 0 + while (choices(index)._1 < l) { + index += 1 + } + val childRowLoc = rowLoc.withNewChild() + choices(index)._2(childRowLoc) + } + + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): JSONChoiceGenFunc = + JSONChoiceGenFunc(choices, lengthGen, mapping) + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): JSONChoiceGenFunc = + JSONChoiceGenFunc(choices, lengthGen, mapping) + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for JSON") +} + +class JSONChoiceGen(val choices: Seq[(Double, String, SubstringDataGen)], + conf: ColumnConf, + defaultValueRange: Option[(Any, Any)] = None) + extends SubstringDataGen(conf, defaultValueRange) { + + override val children: Seq[(String, SubstringDataGen)] = + choices.map { case (_, name, gen) => (name, gen) } + + override def setCorrelatedKeyGroup(keyGroup: Long, + minSeed: Long, maxSeed: Long, + seedMapping: LocationToSeedMapping): SubstringDataGen = { + super.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) + children.foreach { + case (_, gen) => + gen.setCorrelatedKeyGroup(keyGroup, minSeed, maxSeed, seedMapping) + } + this + } + + override def get(name: String): Option[SubstringDataGen] = + children.collectFirst { + case (childName, dataGen) if childName.equalsIgnoreCase(name) => dataGen + } + + override protected def getValGen: GeneratorFunction = { + val childGens = choices.map(c => (c._1, c._3.getGen)).toList + JSONChoiceGenFunc(childGens) + } } case class ASCIIGenFunc( @@ -1672,14 +2451,46 @@ case class ASCIIGenFunc( throw new IllegalArgumentException("value ranges are not supported for strings") } -class StringGen(conf: ColumnConf, defaultValueRange: Option[(Any, Any)]) - extends DataGen(conf, defaultValueRange) { +/** + * This is here to wrap the substring gen function so that its length/settings + * are the ones used when generating a string, and not what was set for the string. + */ +case class SubstringGenFunc( + substringGen: GeneratorFunction, + lengthGen: LengthGeneratorFunction = null, + mapping: LocationToSeedMapping = null) extends GeneratorFunction { + + override def apply(rowLoc: RowLocation): Any = { + substringGen(rowLoc) + } + + // The length and location seed mapping are just ignored for this... + override def withLengthGeneratorFunction(lengthGen: LengthGeneratorFunction): GeneratorFunction = + this + + override def withLocationToSeedMapping(mapping: LocationToSeedMapping): GeneratorFunction = + this + + override def withValueRange(min: Any, max: Any): GeneratorFunction = + throw new IllegalArgumentException("value ranges are not supported for strings") +} + +class StringGen(conf: ColumnConf, + defaultValueRange: Option[(Any, Any)], + var substringDataGen: Option[SubstringDataGen] = None) + extends DataGen(conf, defaultValueRange) { override def dataType: DataType = StringType - override protected def getValGen: GeneratorFunction = ASCIIGenFunc() + override protected def getValGen: GeneratorFunction = + substringDataGen.map(s => SubstringGenFunc(s.getGen)).getOrElse(ASCIIGenFunc()) override def children: Seq[(String, DataGen)] = Seq.empty + + override def setSubstringGen(subgen: Option[SubstringDataGen]): Unit = + substringDataGen = subgen + + override def getSubstringGen: Option[SubstringDataGen] = substringDataGen } case class StructGenFunc(childGens: Array[GeneratorFunction]) extends GeneratorFunction { @@ -1854,7 +2665,6 @@ class MapGen(key: DataGen, override def children: Seq[(String, DataGen)] = Seq(("key", key), ("value", value)) } - object ColumnGen { private def genInternal(rowNumber: Column, dataType: DataType, @@ -1869,8 +2679,8 @@ object ColumnGen { */ class ColumnGen(val dataGen: DataGen) { def setCorrelatedKeyGroup(kg: Long, - minSeed: Long, maxSeed: Long, - seedMapping: LocationToSeedMapping): ColumnGen = { + minSeed: Long, maxSeed: Long, + seedMapping: LocationToSeedMapping): ColumnGen = { dataGen.setCorrelatedKeyGroup(kg, minSeed, maxSeed, seedMapping) this } @@ -1930,6 +2740,11 @@ class ColumnGen(val dataGen: DataGen) { this } + def setGaussianLength(mean: Double, stdDev: Double): ColumnGen = { + dataGen.setGaussianLength(mean, stdDev) + this + } + final def apply(name: String): DataGen = { get(name).getOrElse { throw new IllegalArgumentException(s"$name not a child of $this") @@ -1941,8 +2756,16 @@ class ColumnGen(val dataGen: DataGen) { def gen(rowNumber: Column): Column = { ColumnGen.genInternal(rowNumber, dataGen.dataType, dataGen.nullable, dataGen.getGen) } + + def getSubstring: Option[SubstringDataGen] = dataGen.getSubstringGen + + def substringGen: SubstringDataGen = dataGen.substringGen + + def setSubstringGen(f : ColumnConf => SubstringDataGen): Unit = + dataGen.setSubstringGen(f) } + sealed trait KeyGroupType /** @@ -2192,7 +3015,7 @@ object DBGen { numRows: Long, mapping: OrderedTypeMapping): Seq[(String, ColumnGen)] = { // a bit of a hack with the column num so that we update it before each time... - var conf = ColumnConf(ColumnLocation(tableId, -1), true, numRows) + var conf = ColumnConf(ColumnLocation(tableId, -1, 0), true, numRows) st.toArray.map { sf => if (!mapping.canMap(sf.dataType, mapping)) { throw new IllegalArgumentException(s"$sf is not supported at this time") diff --git a/jenkins/databricks/install_deps.py b/jenkins/databricks/install_deps.py index be5cb9bc040..8d21a4f9556 100644 --- a/jenkins/databricks/install_deps.py +++ b/jenkins/databricks/install_deps.py @@ -115,8 +115,10 @@ def define_deps(spark_version, scala_version): f'{prefix_ws_sp_mvn_hadoop}--org.json4s--json4s-jackson_{scala_version}--org.json4s__json4s-jackson_{scala_version}__*.jar'), Artifact('org.javaassist', 'javaassist', f'{prefix_ws_sp_mvn_hadoop}--org.javassist--javassist--org.javassist__javassist__*.jar'), - Artifact('com.fasterxml.jackson.core', 'jackson-core', + Artifact('com.fasterxml.jackson.core', 'jackson-databind', f'{prefix_ws_sp_mvn_hadoop}--com.fasterxml.jackson.core--jackson-databind--com.fasterxml.jackson.core__jackson-databind__*.jar'), + Artifact('com.fasterxml.jackson.core', 'jackson-core', + f'{prefix_ws_sp_mvn_hadoop}--com.fasterxml.jackson.core--jackson-core--com.fasterxml.jackson.core__jackson-core__*.jar'), Artifact('com.fasterxml.jackson.core', 'jackson-annotations', f'{prefix_ws_sp_mvn_hadoop}--com.fasterxml.jackson.core--jackson-annotations--com.fasterxml.jackson.core__jackson-annotations__*.jar'), Artifact('org.apache.spark', f'spark-avro_{scala_version}', diff --git a/scala2.13/shim-deps/databricks/pom.xml b/scala2.13/shim-deps/databricks/pom.xml index b342f381c71..a0459901079 100644 --- a/scala2.13/shim-deps/databricks/pom.xml +++ b/scala2.13/shim-deps/databricks/pom.xml @@ -105,6 +105,12 @@ ${spark.version} compile + + com.fasterxml.jackson.core + jackson-databind + ${spark.version} + compile + com.fasterxml.jackson.core jackson-annotations @@ -286,4 +292,4 @@ compile - \ No newline at end of file + diff --git a/shim-deps/databricks/pom.xml b/shim-deps/databricks/pom.xml index bef8a90d227..22842b0f7c0 100644 --- a/shim-deps/databricks/pom.xml +++ b/shim-deps/databricks/pom.xml @@ -105,6 +105,12 @@ ${spark.version} compile + + com.fasterxml.jackson.core + jackson-databind + ${spark.version} + compile + com.fasterxml.jackson.core jackson-annotations @@ -286,4 +292,4 @@ compile - \ No newline at end of file + From eb1549c5764e2ceaaac1d9f7f885c0d6c11ae92c Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Thu, 13 Jun 2024 14:23:57 -0700 Subject: [PATCH 33/79] `binary-dedupe` changes for Spark 4.0.0 [databricks] (#10993) * Binary dedupe changes for Spark 4.0.0 Signed-off-by: Raza Jafri * updated comments * Changed the URL for the common classes among shims * renamed spark34-common to spark-shared and renamed relevant variables * addressed review comments * renamed variable from common to shared --------- Signed-off-by: Raza Jafri --- dist/maven-antrun/build-parallel-worlds.xml | 4 +- dist/scripts/binary-dedupe.sh | 58 +++++++++---------- .../com/nvidia/spark/rapids/ShimLoader.scala | 10 ++-- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/dist/maven-antrun/build-parallel-worlds.xml b/dist/maven-antrun/build-parallel-worlds.xml index 524b15addf9..07838616340 100644 --- a/dist/maven-antrun/build-parallel-worlds.xml +++ b/dist/maven-antrun/build-parallel-worlds.xml @@ -1,6 +1,6 @@ - diff --git a/dist/scripts/binary-dedupe.sh b/dist/scripts/binary-dedupe.sh index 183e86b1524..356b0b4dbae 100755 --- a/dist/scripts/binary-dedupe.sh +++ b/dist/scripts/binary-dedupe.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -34,10 +34,10 @@ case "$OSTYPE" in esac STEP=0 -export SPARK3XX_COMMON_TXT="$PWD/spark3xx-common.txt" -export SPARK3XX_COMMON_COPY_LIST="$PWD/spark-common-copy-list.txt" +export SPARK_SHARED_TXT="$PWD/spark-shared.txt" +export SPARK_SHARED_COPY_LIST="$PWD/spark-shared-copy-list.txt" export DELETE_DUPLICATES_TXT="$PWD/delete-duplicates.txt" -export SPARK3XX_COMMON_DIR="$PWD/spark3xx-common" +export SPARK_SHARED_DIR="$PWD/spark-shared" # This script de-duplicates .class files at the binary level. # We could also diff classes using scalap / javap outputs. @@ -47,17 +47,17 @@ export SPARK3XX_COMMON_DIR="$PWD/spark3xx-common" # The following pipeline determines identical classes across shims in this build. # - checksum all class files -# - move the varying-prefix spark3xy to the left so it can be easily skipped for uniq and sort +# - move the varying-prefix sparkxyz to the left so it can be easily skipped for uniq and sort # - sort by path, secondary sort by checksum, print one line per group # - produce uniq count for paths # - filter the paths with count=1, the class files without diverging checksums -# - put the path starting with /spark3xy back together for the final list +# - put the path starting with /sparkxyz back together for the final list echo "Retrieving class files hashing to a single value ..." echo "$((++STEP))/ SHA1 of all non-META files > tmp-sha1-files.txt" -find ./parallel-world/spark3* -name META-INF -prune -o \( -type f -print \) | \ - xargs $SHASUM > tmp-sha1-files.txt +find ./parallel-world/spark[34]* -name META-INF -prune -o -name webapps -prune -o \( -type f -print0 \) | \ + xargs --null $SHASUM > tmp-sha1-files.txt echo "$((++STEP))/ make shim column 1 > tmp-shim-sha-package-files.txt" < tmp-sha1-files.txt awk -F/ '$1=$1' | \ @@ -68,10 +68,10 @@ echo "$((++STEP))/ sort by path, sha1; output first from each group > tmp-count- sort -k3 -k2,2 -u tmp-shim-sha-package-files.txt | \ uniq -f 2 -c > tmp-count-shim-sha-package-files.txt -echo "$((++STEP))/ files with unique sha1 > $SPARK3XX_COMMON_TXT" +echo "$((++STEP))/ files with unique sha1 > $SPARK_SHARED_TXT" grep '^\s\+1 .*' tmp-count-shim-sha-package-files.txt | \ awk '{$1=""; $3=""; print $0 }' | \ - tr -s ' ' | sed 's/\ /\//g' > "$SPARK3XX_COMMON_TXT" + tr -s ' ' | sed 's/\ /\//g' > "$SPARK_SHARED_TXT" function retain_single_copy() { set -e @@ -93,10 +93,10 @@ function retain_single_copy() { package_class="${package_class_with_spaces// //}" # get the reference copy out of the way - echo "$package_class" >> "from-$shim-to-spark3xx-common.txt" + echo "$package_class" >> "from-$shim-to-spark-shared.txt" # expanding directories separately because full path # glob is broken for class file name including the "$" character - for pw in ./parallel-world/spark3* ; do + for pw in ./parallel-world/spark[34]* ; do delete_path="$pw/$package_class" [[ -f "$delete_path" ]] && echo "$delete_path" || true done >> "$DELETE_DUPLICATES_TXT" || exit 255 @@ -106,26 +106,26 @@ function retain_single_copy() { # standalone debugging # truncate incremental files : > "$DELETE_DUPLICATES_TXT" -rm -f from-spark3*-to-spark3xx-common.txt -rm -rf "$SPARK3XX_COMMON_DIR" -mkdir -p "$SPARK3XX_COMMON_DIR" +rm -f from-spark[34]*-to-spark-shared.txt +rm -rf "$SPARK_SHARED_DIR" +mkdir -p "$SPARK_SHARED_DIR" -echo "$((++STEP))/ retaining a single copy of spark3xx-common classes" +echo "$((++STEP))/ retaining a single copy of spark-shared classes" while read spark_common_class; do retain_single_copy "$spark_common_class" -done < "$SPARK3XX_COMMON_TXT" +done < "$SPARK_SHARED_TXT" -echo "$((++STEP))/ rsyncing common classes to $SPARK3XX_COMMON_DIR" -for copy_list in from-spark3*-to-spark3xx-common.txt; do +echo "$((++STEP))/ rsyncing common classes to $SPARK_SHARED_DIR" +for copy_list in from-spark[34]*-to-spark-shared.txt; do echo Initializing rsync of "$copy_list" IFS='-' <<< "$copy_list" read -ra copy_list_parts # declare -p copy_list_parts shim="${copy_list_parts[1]}" # use rsync to reduce process forking - rsync --files-from="$copy_list" ./parallel-world/"$shim" "$SPARK3XX_COMMON_DIR" + rsync --files-from="$copy_list" ./parallel-world/"$shim" "$SPARK_SHARED_DIR" done -mv "$SPARK3XX_COMMON_DIR" parallel-world/ +mv "$SPARK_SHARED_DIR" parallel-world/ # TODO further dedupe by FEATURE version lines: # spark30x-common @@ -137,9 +137,9 @@ mv "$SPARK3XX_COMMON_DIR" parallel-world/ # # At this point the duplicate classes have not been removed from version-specific jar # locations such as parallel-world/spark312. -# For each unshimmed class file look for all of its copies inside /spark3* and +# For each unshimmed class file look for all of its copies inside /spark[34]* and # and count the number of distinct checksums. There are two representative cases -# 1) The class is contributed to the unshimmed location via the unshimmed-from-each-spark3xx list. These are classes +# 1) The class is contributed to the unshimmed location via the unshimmed-from-each-spark34 list. These are classes # carrying the shim classifier in their package name such as # com.nvidia.spark.rapids.spark312.RapidsShuffleManager. They are unique by construction, # and will have zero copies in any non-spark312 shims. Although such classes are currently excluded from @@ -157,25 +157,25 @@ mv "$SPARK3XX_COMMON_DIR" parallel-world/ # Determine the list of unshimmed class files UNSHIMMED_LIST_TXT=unshimmed-result.txt echo "$((++STEP))/ creating sorted list of unshimmed classes > $UNSHIMMED_LIST_TXT" -find ./parallel-world -name '*.class' -not -path './parallel-world/spark3*' | \ +find ./parallel-world -name '*.class' -not -path './parallel-world/spark[34-]*' | \ cut -d/ -f 3- | sort > "$UNSHIMMED_LIST_TXT" function verify_same_sha_for_unshimmed() { set -e class_file="$1" - # the raw spark3xx-common.txt file list contains all single-sha1 classes + # the raw spark-shared.txt file list contains all single-sha1 classes # including the ones that are unshimmed. Instead of expensively recomputing # sha1 look up if there is an entry with the unshimmed class as a suffix class_file_quoted=$(printf '%q' "$class_file") - # TODO currently RapidsShuffleManager is "removed" from /spark3* by construction in + # TODO currently RapidsShuffleManager is "removed" from /spark* by construction in # dist pom.xml via ant. We could delegate this logic to this script # and make both simmpler - if [[ ! "$class_file_quoted" =~ (com/nvidia/spark/rapids/spark3.*/.*ShuffleManager.class|org/apache/spark/sql/rapids/shims/spark3.*/ProxyRapidsShuffleInternalManager.class) ]]; then + if [[ ! "$class_file_quoted" =~ (com/nvidia/spark/rapids/spark[34].*/.*ShuffleManager.class|org/apache/spark/sql/rapids/shims/spark[34].*/ProxyRapidsShuffleInternalManager.class) ]]; then - if ! grep -q "/spark.\+/$class_file_quoted" "$SPARK3XX_COMMON_TXT"; then + if ! grep -q "/spark.\+/$class_file_quoted" "$SPARK_SHARED_TXT"; then echo >&2 "$class_file is not bitwise-identical across shims" exit 255 fi @@ -192,7 +192,7 @@ done < "$UNSHIMMED_LIST_TXT" echo "$((++STEP))/ removing duplicates of unshimmed classes" while read unshimmed_class; do - for pw in ./parallel-world/spark3* ; do + for pw in ./parallel-world/spark[34]* ; do unshimmed_path="$pw/$unshimmed_class" [[ -f "$unshimmed_path" ]] && echo "$unshimmed_path" || true done >> "$DELETE_DUPLICATES_TXT" diff --git a/sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala b/sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala index 36abc75ba87..2d7a51c4e43 100644 --- a/sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala +++ b/sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala @@ -40,19 +40,19 @@ import org.apache.spark.util.MutableURLClassLoader "parallel worlds" in the JDK's com.sun.istack.internal.tools.ParallelWorldClassLoader parlance 1. a few publicly documented classes in the conventional layout at the top 2. a large fraction of classes whose bytecode is identical under all supported Spark versions - in spark3xx-common + in spark-shared 3. a smaller fraction of classes that differ under one of the supported Spark versions com/nvidia/spark/SQLPlugin.class - spark3xx-common/com/nvidia/spark/rapids/CastExprMeta.class + spark-shared/com/nvidia/spark/rapids/CastExprMeta.class spark311/org/apache/spark/sql/rapids/GpuUnaryMinus.class spark320/org/apache/spark/sql/rapids/GpuUnaryMinus.class Each shim can see a consistent parallel world without conflicts by referencing only one conflicting directory. E.g., Spark 3.2.0 Shim will use - jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark3xx-common/ + jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark-shared/ jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark320/ Spark 3.1.1 will use - jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark3xx-common/ + jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark-shared/ jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark311/ Using these Jar URL's allows referencing different bytecode produced from identical sources by incompatible Scala / Spark dependencies. @@ -67,7 +67,7 @@ object ShimLoader extends Logging { new URL(rootUrlStr) } - private val shimCommonURL = new URL(s"${shimRootURL.toString}spark3xx-common/") + private val shimCommonURL = new URL(s"${shimRootURL.toString}spark-shared/") @volatile private var shimProviderClass: String = _ @volatile private var shimProvider: SparkShimServiceProvider = _ @volatile private var shimURL: URL = _ From 356d5a1a5021c421cef7fbb73734527ba0ac1857 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Fri, 14 Jun 2024 08:45:28 +0800 Subject: [PATCH 34/79] [FEA] Increase parallelism of deltalake test on databricks (#11051) --- integration_tests/run_pyspark_from_build.sh | 6 ++++++ jenkins/databricks/test.sh | 5 +---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh index dec93e6f22a..8b10b3debac 100755 --- a/integration_tests/run_pyspark_from_build.sh +++ b/integration_tests/run_pyspark_from_build.sh @@ -245,6 +245,12 @@ else DRIVER_EXTRA_JAVA_OPTIONS="-ea -Duser.timezone=$TZ -Ddelta.log.cacheSize=$deltaCacheSize" export PYSP_TEST_spark_driver_extraJavaOptions="$DRIVER_EXTRA_JAVA_OPTIONS $COVERAGE_SUBMIT_FLAGS" export PYSP_TEST_spark_executor_extraJavaOptions="-ea -Duser.timezone=$TZ" + + # Set driver memory to speed up tests such as deltalake + if [[ -n "${DRIVER_MEMORY}" ]]; then + export PYSP_TEST_spark_driver_memory="${DRIVER_MEMORY}" + fi + export PYSP_TEST_spark_ui_showConsoleProgress='false' export PYSP_TEST_spark_sql_session_timeZone=$TZ export PYSP_TEST_spark_sql_shuffle_partitions='4' diff --git a/jenkins/databricks/test.sh b/jenkins/databricks/test.sh index f71f69844f7..c966d5a92f7 100755 --- a/jenkins/databricks/test.sh +++ b/jenkins/databricks/test.sh @@ -66,9 +66,6 @@ TEST_MODE=${TEST_MODE:-'DEFAULT'} # --packages in distributed setups, should be fixed by # https://github.com/NVIDIA/spark-rapids/pull/5646 -# Increase driver memory as Delta Lake tests can slowdown with default 1G (possibly due to caching?) -DELTA_LAKE_CONFS="--driver-memory 2g" - # Enable event log for qualification & profiling tools testing export PYSP_TEST_spark_eventLog_enabled=true mkdir -p /tmp/spark-events @@ -105,7 +102,7 @@ if [[ "$(pwd)" == "$SOURCE_PATH" ]]; then if [[ "$TEST_MODE" == "DEFAULT" || $TEST_MODE == "CI_PART2" || "$TEST_MODE" == "DELTA_LAKE_ONLY" ]]; then ## Run Delta Lake tests - SPARK_SUBMIT_FLAGS="$SPARK_CONF $DELTA_LAKE_CONFS" TEST_PARALLEL=1 \ + DRIVER_MEMORY="4g" \ bash integration_tests/run_pyspark_from_build.sh --runtime_env="databricks" -m "delta_lake" --delta_lake --test_type=$TEST_TYPE fi From 599ae17e02c8b88c63fe0133caf19570f332092e Mon Sep 17 00:00:00 2001 From: "Hongbin Ma (Mahone)" Date: Fri, 14 Jun 2024 09:27:03 +0800 Subject: [PATCH 35/79] fix flaky array_item test failures (#11054) * fix flaky array_item test failures Signed-off-by: Hongbin Ma (Mahone) * fix indent Signed-off-by: Hongbin Ma (Mahone) * fix whitespace Signed-off-by: Hongbin Ma (Mahone) --------- Signed-off-by: Hongbin Ma (Mahone) --- integration_tests/src/main/python/data_gen.py | 23 ++++++++++++------- .../src/main/python/parquet_write_test.py | 6 ++++- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/integration_tests/src/main/python/data_gen.py b/integration_tests/src/main/python/data_gen.py index 2e6c36b77d9..fb1627af75b 100644 --- a/integration_tests/src/main/python/data_gen.py +++ b/integration_tests/src/main/python/data_gen.py @@ -159,7 +159,8 @@ def __repr__(self): return super().__repr__() + '(' + str(self._child_gen) + ')' def _cache_repr(self): - return super()._cache_repr() + '(' + self._child_gen._cache_repr() + ')' + return (super()._cache_repr() + '(' + self._child_gen._cache_repr() + + ',' + str(self._func.__code__) + ')' ) def start(self, rand): self._child_gen.start(rand) @@ -667,7 +668,10 @@ def __repr__(self): return super().__repr__() + '(' + str(self._child_gen) + ')' def _cache_repr(self): - return super()._cache_repr() + '(' + self._child_gen._cache_repr() + ')' + return (super()._cache_repr() + '(' + self._child_gen._cache_repr() + + ',' + str(self._min_length) + ',' + str(self._max_length) + ',' + + str(self.all_null) + ',' + str(self.convert_to_tuple) + ')') + def start(self, rand): self._child_gen.start(rand) @@ -701,7 +705,8 @@ def __repr__(self): return super().__repr__() + '(' + str(self._key_gen) + ',' + str(self._value_gen) + ')' def _cache_repr(self): - return super()._cache_repr() + '(' + self._key_gen._cache_repr() + ',' + self._value_gen._cache_repr() + ')' + return (super()._cache_repr() + '(' + self._key_gen._cache_repr() + ',' + self._value_gen._cache_repr() + + ',' + str(self._min_length) + ',' + str(self._max_length) + ')') def start(self, rand): self._key_gen.start(rand) @@ -769,12 +774,13 @@ def __init__(self, min_value=MIN_DAY_TIME_INTERVAL, max_value=MAX_DAY_TIME_INTER self._min_micros = (math.floor(min_value.total_seconds()) * 1000000) + min_value.microseconds self._max_micros = (math.floor(max_value.total_seconds()) * 1000000) + max_value.microseconds fields = ["day", "hour", "minute", "second"] - start_index = fields.index(start_field) - end_index = fields.index(end_field) - if start_index > end_index: + self._start_index = fields.index(start_field) + self._end_index = fields.index(end_field) + if self._start_index > self._end_index: raise RuntimeError('Start field {}, end field {}, valid fields is {}, start field index should <= end ' 'field index'.format(start_field, end_field, fields)) - super().__init__(DayTimeIntervalType(start_index, end_index), nullable=nullable, special_cases=special_cases) + super().__init__(DayTimeIntervalType(self._start_index, self._end_index), nullable=nullable, + special_cases=special_cases) def _gen_random(self, rand): micros = rand.randint(self._min_micros, self._max_micros) @@ -784,7 +790,8 @@ def _gen_random(self, rand): return timedelta(microseconds=micros) def _cache_repr(self): - return super()._cache_repr() + '(' + str(self._min_micros) + ',' + str(self._max_micros) + ')' + return (super()._cache_repr() + '(' + str(self._min_micros) + ',' + str(self._max_micros) + + ',' + str(self._start_index) + ',' + str(self._end_index) + ')') def start(self, rand): self._start(rand, lambda: self._gen_random(rand)) diff --git a/integration_tests/src/main/python/parquet_write_test.py b/integration_tests/src/main/python/parquet_write_test.py index 99a2d4241e8..38dab9e84a4 100644 --- a/integration_tests/src/main/python/parquet_write_test.py +++ b/integration_tests/src/main/python/parquet_write_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -224,6 +224,10 @@ def test_all_null_int96(spark_tmp_path): class AllNullTimestampGen(TimestampGen): def start(self, rand): self._start(rand, lambda : None) + + def _cache_repr(self): + return super()._cache_repr() + '(all_nulls)' + data_path = spark_tmp_path + '/PARQUET_DATA' confs = copy_and_update(writer_confs, {'spark.sql.parquet.outputTimestampType': 'INT96'}) assert_gpu_and_cpu_writes_are_equal_collect( From 2f3c0c276dba3e8870f3e3bfa9287e644809f4f6 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Fri, 14 Jun 2024 19:47:56 +0800 Subject: [PATCH 36/79] Calculate parallelism to speed up pre-merge CI (#11046) * Calculate parallelism to speed up pre-merge CI Calculate parallelism based on GPU memory to speed up pre-merge CI with appropriate amount of parallelism. But when TEST_PARALLEL > 8 and as it increases, the integration tests running speed will become slower and slower, so we limit TEST_PARALLEL <= 8. Based on this change, and ran pre-merge CI on powerful nodes, we observed the pre-merge CI 1 hour less than on common nodes. 16 CPU/128G Mem/24G GPU : [2hours] VS 8 CPU/64G Mem/16G GPU : [3hours] Note: currently we only have 3 fixed powerful nodes for the pre-merge CI job, so only 1 pre-merge CI be speeded up at the same time Signed-off-by: Tim Liu * Add a variable to set maximum test parallelism for the integration tests Signed-off-by: Tim Liu * Fix typo Signed-off-by: Tim Liu --------- Signed-off-by: Tim Liu --- integration_tests/run_pyspark_from_build.sh | 5 +++++ jenkins/spark-premerge-build.sh | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh index 8b10b3debac..18c26aa26e7 100755 --- a/integration_tests/run_pyspark_from_build.sh +++ b/integration_tests/run_pyspark_from_build.sh @@ -171,11 +171,16 @@ else TEST_TYPE_PARAM="--test_type $TEST_TYPE" fi + # We found that when parallelism > 8, as it increases, the test speed will become slower and slower. So we set the default maximum parallelism to 8. + # Note that MAX_PARALLEL varies with the hardware, OS, and test case. Please overwrite it with an appropriate value if needed. + MAX_PARALLEL=${MAX_PARALLEL:-8} if [[ ${TEST_PARALLEL} -lt 2 ]]; then # With xdist 0 and 1 are the same parallelism but # 0 is more efficient TEST_PARALLEL_OPTS=() + elif [[ ${TEST_PARALLEL} -gt ${MAX_PARALLEL} ]]; then + TEST_PARALLEL_OPTS=("-n" "$MAX_PARALLEL") else TEST_PARALLEL_OPTS=("-n" "$TEST_PARALLEL") fi diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh index 883b3f3acfc..697722c0138 100755 --- a/jenkins/spark-premerge-build.sh +++ b/jenkins/spark-premerge-build.sh @@ -78,7 +78,7 @@ mvn_verify() { # Here run Python integration tests tagged with 'premerge_ci_1' only, that would help balance test duration and memory # consumption from two k8s pods running in parallel, which executes 'mvn_verify()' and 'ci_2()' respectively. $MVN_CMD -B $MVN_URM_MIRROR $PREMERGE_PROFILES clean verify -Dpytest.TEST_TAGS="premerge_ci_1" \ - -Dpytest.TEST_TYPE="pre-commit" -Dpytest.TEST_PARALLEL=4 -Dcuda.version=$CLASSIFIER + -Dpytest.TEST_TYPE="pre-commit" -Dcuda.version=$CLASSIFIER # The jacoco coverage should have been collected, but because of how the shade plugin # works and jacoco we need to clean some things up so jacoco will only report for the @@ -162,7 +162,6 @@ ci_2() { $MVN_CMD -U -B $MVN_URM_MIRROR clean package $MVN_BUILD_ARGS -DskipTests=true export TEST_TAGS="not premerge_ci_1" export TEST_TYPE="pre-commit" - export TEST_PARALLEL=5 # Download a Scala 2.12 build of spark prepare_spark $SPARK_VER 2.12 @@ -206,7 +205,6 @@ ci_scala213() { cd .. # Run integration tests in the project root dir to leverage test cases and resource files export TEST_TAGS="not premerge_ci_1" export TEST_TYPE="pre-commit" - export TEST_PARALLEL=5 # SPARK_HOME (and related) must be set to a Spark built with Scala 2.13 SPARK_HOME=$SPARK_HOME PYTHONPATH=$PYTHONPATH \ ./integration_tests/run_pyspark_from_build.sh From 6eb854dc82a4ec0e96b97871a97c6b9ed7471722 Mon Sep 17 00:00:00 2001 From: Peixin Date: Mon, 17 Jun 2024 18:12:49 +0800 Subject: [PATCH 37/79] WAR numpy2 failed fastparquet compatibility issue (#11072) Signed-off-by: Peixin Li --- .../src/main/python/fastparquet_compatibility_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration_tests/src/main/python/fastparquet_compatibility_test.py b/integration_tests/src/main/python/fastparquet_compatibility_test.py index 53a99d32bd2..4b0fc2827f4 100644 --- a/integration_tests/src/main/python/fastparquet_compatibility_test.py +++ b/integration_tests/src/main/python/fastparquet_compatibility_test.py @@ -30,6 +30,8 @@ def fastparquet_unavailable(): return False except ImportError: return True + except ValueError: # TODO: remove when https://github.com/NVIDIA/spark-rapids/issues/11070 is fixed + return True rebase_write_corrected_conf = { From 0952dea254df6fc4f1f01a9e0e8ac50f97285233 Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Tue, 18 Jun 2024 14:24:16 +0800 Subject: [PATCH 38/79] Fallback non-UTC TimeZoneAwareExpression with zoneId [databricks] (#10996) * Fallback non-UTC TimeZoneAwareExpression with zoneId instead of timeZone config Signed-off-by: Haoyang Li * clean up Signed-off-by: Haoyang Li --------- Signed-off-by: Haoyang Li --- .../main/scala/com/nvidia/spark/rapids/RapidsMeta.scala | 8 ++++---- .../spark/sql/rapids/utils/RapidsTestSettings.scala | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala index a876ea6c9e0..984892cd787 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsMeta.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -1123,7 +1123,7 @@ abstract class BaseExprMeta[INPUT <: Expression]( if (!needTimeZoneCheck) return // Level 2 check - if (!isTimeZoneSupported) return checkUTCTimezone(this) + if (!isTimeZoneSupported) return checkUTCTimezone(this, getZoneId()) // Level 3 check val zoneId = getZoneId() @@ -1203,8 +1203,8 @@ abstract class BaseExprMeta[INPUT <: Expression]( * * @param meta to check whether it's UTC */ - def checkUTCTimezone(meta: RapidsMeta[_, _, _]): Unit = { - if (!GpuOverrides.isUTCTimezone()) { + def checkUTCTimezone(meta: RapidsMeta[_, _, _], zoneId: ZoneId): Unit = { + if (!GpuOverrides.isUTCTimezone(zoneId)) { meta.willNotWorkOnGpu( TimeZoneDB.nonUTCTimezoneNotSupportedStr(meta.wrapped.getClass.toString)) } diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala index 4cf155041d9..63649376829 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala @@ -48,11 +48,9 @@ class RapidsTestSettings extends BackendTestSettings { .exclude("from_json - input=empty array, schema=struct, output=single row with null", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) .exclude("from_json - input=empty object, schema=struct, output=single row with null", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) .exclude("SPARK-20549: from_json bad UTF-8", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("from_json with timestamp", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) .exclude("to_json - array", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) .exclude("to_json - array with single empty row", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) .exclude("to_json - empty array", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("to_json with timestamp", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) .exclude("SPARK-21513: to_json support map[string, struct] to json", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) .exclude("SPARK-21513: to_json support map[struct, struct] to json", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) .exclude("SPARK-21513: to_json support map[string, integer] to json", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) From 7bac3a6439c10efb1961d3c4ba028128d9dca249 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 19 Jun 2024 09:44:48 +0800 Subject: [PATCH 39/79] [FEA] Introduce low shuffle merge. (#10979) * feat: Introduce low shuffle merge. Signed-off-by: liurenjie1024 * fix * Test databricks parallel * Test more databricks parallel * Fix comments * Config && scala 2.13 * Revert * Fix comments * scala 2.13 * Revert unnecessary changes * Revert "Revert unnecessary changes" This reverts commit 9fa4cf268cc3fce4d2732e04cb33eb53e4859c99. * restore change --------- Signed-off-by: liurenjie1024 --- aggregator/pom.xml | 4 + .../GpuDeltaParquetFileFormatUtils.scala | 160 +++ .../nvidia/spark/rapids/delta/deltaUDFs.scala | 83 +- .../delta/delta24x/Delta24xProvider.scala | 5 +- .../GpuDelta24xParquetFileFormat.scala | 61 +- .../delta/delta24x/MergeIntoCommandMeta.scala | 58 +- .../delta24x/GpuLowShuffleMergeCommand.scala | 1084 +++++++++++++++++ .../rapids/GpuLowShuffleMergeCommand.scala | 1083 ++++++++++++++++ .../delta/GpuDeltaParquetFileFormat.scala | 63 +- .../shims/MergeIntoCommandMetaShim.scala | 101 +- .../advanced_configs.md | 6 + .../delta_lake_low_shuffle_merge_test.py | 165 +++ .../main/python/delta_lake_merge_common.py | 155 +++ .../src/main/python/delta_lake_merge_test.py | 127 +- pom.xml | 10 + scala2.13/aggregator/pom.xml | 4 + scala2.13/pom.xml | 10 + scala2.13/sql-plugin/pom.xml | 4 + sql-plugin/pom.xml | 4 + .../com/nvidia/spark/rapids/RapidsConf.scala | 28 + 20 files changed, 3061 insertions(+), 154 deletions(-) create mode 100644 delta-lake/common/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormatUtils.scala create mode 100644 delta-lake/delta-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/GpuLowShuffleMergeCommand.scala create mode 100644 delta-lake/delta-spark341db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuLowShuffleMergeCommand.scala create mode 100644 integration_tests/src/main/python/delta_lake_low_shuffle_merge_test.py create mode 100644 integration_tests/src/main/python/delta_lake_merge_common.py diff --git a/aggregator/pom.xml b/aggregator/pom.xml index 22bfe11105e..8cf881419c9 100644 --- a/aggregator/pom.xml +++ b/aggregator/pom.xml @@ -94,6 +94,10 @@ com.google.flatbuffers ${rapids.shade.package}.com.google.flatbuffers + + org.roaringbitmap + ${rapids.shade.package}.org.roaringbitmap + diff --git a/delta-lake/common/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormatUtils.scala b/delta-lake/common/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormatUtils.scala new file mode 100644 index 00000000000..101a82da830 --- /dev/null +++ b/delta-lake/common/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormatUtils.scala @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.delta + +import ai.rapids.cudf.{ColumnVector => CudfColumnVector, Scalar, Table} +import com.nvidia.spark.rapids.Arm.{closeOnExcept, withResource} +import com.nvidia.spark.rapids.GpuColumnVector +import org.roaringbitmap.longlong.{PeekableLongIterator, Roaring64Bitmap} + +import org.apache.spark.sql.types.{BooleanType, LongType, StringType, StructField, StructType} +import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} + + +object GpuDeltaParquetFileFormatUtils { + /** + * Row number of the row in the file. When used with [[FILE_PATH_COL]] together, it can be used + * as unique id of a row in file. Currently to correctly calculate this, the caller needs to + * set both [[isSplitable]] to false, and [[RapidsConf.PARQUET_READER_TYPE]] to "PERFILE". + */ + val METADATA_ROW_IDX_COL: String = "__metadata_row_index" + val METADATA_ROW_IDX_FIELD: StructField = StructField(METADATA_ROW_IDX_COL, LongType, + nullable = false) + + val METADATA_ROW_DEL_COL: String = "__metadata_row_del" + val METADATA_ROW_DEL_FIELD: StructField = StructField(METADATA_ROW_DEL_COL, BooleanType, + nullable = false) + + + /** + * File path of the file that the row came from. + */ + val FILE_PATH_COL: String = "_metadata_file_path" + val FILE_PATH_FIELD: StructField = StructField(FILE_PATH_COL, StringType, nullable = false) + + /** + * Add a metadata column to the iterator. Currently only support [[METADATA_ROW_IDX_COL]]. + */ + def addMetadataColumnToIterator( + schema: StructType, + delVector: Option[Roaring64Bitmap], + input: Iterator[ColumnarBatch], + maxBatchSize: Int): Iterator[ColumnarBatch] = { + val metadataRowIndexCol = schema.fieldNames.indexOf(METADATA_ROW_IDX_COL) + val delRowIdx = schema.fieldNames.indexOf(METADATA_ROW_DEL_COL) + if (metadataRowIndexCol == -1 && delRowIdx == -1) { + return input + } + var rowIndex = 0L + input.map { batch => + withResource(batch) { _ => + val rowIdxCol = if (metadataRowIndexCol == -1) { + None + } else { + Some(metadataRowIndexCol) + } + + val delRowIdx2 = if (delRowIdx == -1) { + None + } else { + Some(delRowIdx) + } + val newBatch = addMetadataColumns(rowIdxCol, delRowIdx2, delVector,maxBatchSize, + rowIndex, batch) + rowIndex += batch.numRows() + newBatch + } + } + } + + private def addMetadataColumns( + rowIdxPos: Option[Int], + delRowIdx: Option[Int], + delVec: Option[Roaring64Bitmap], + maxBatchSize: Int, + rowIdxStart: Long, + batch: ColumnarBatch): ColumnarBatch = { + val rowIdxCol = rowIdxPos.map { _ => + withResource(Scalar.fromLong(rowIdxStart)) { start => + GpuColumnVector.from(CudfColumnVector.sequence(start, batch.numRows()), + METADATA_ROW_IDX_FIELD.dataType) + } + } + + closeOnExcept(rowIdxCol) { rowIdxCol => + + val delVecCol = delVec.map { delVec => + withResource(Scalar.fromBool(false)) { s => + withResource(CudfColumnVector.fromScalar(s, batch.numRows())) { c => + var table = new Table(c) + val posIter = new RoaringBitmapIterator( + delVec.getLongIteratorFrom(rowIdxStart), + rowIdxStart, + rowIdxStart + batch.numRows(), + ).grouped(Math.min(maxBatchSize, batch.numRows())) + + for (posChunk <- posIter) { + withResource(CudfColumnVector.fromLongs(posChunk: _*)) { poses => + withResource(Scalar.fromBool(true)) { s => + table = withResource(table) { _ => + Table.scatter(Array(s), poses, table) + } + } + } + } + + withResource(table) { _ => + GpuColumnVector.from(table.getColumn(0).incRefCount(), + METADATA_ROW_DEL_FIELD.dataType) + } + } + } + } + + closeOnExcept(delVecCol) { delVecCol => + // Replace row_idx column + val columns = new Array[ColumnVector](batch.numCols()) + for (i <- 0 until batch.numCols()) { + if (rowIdxPos.contains(i)) { + columns(i) = rowIdxCol.get + } else if (delRowIdx.contains(i)) { + columns(i) = delVecCol.get + } else { + columns(i) = batch.column(i) match { + case gpuCol: GpuColumnVector => gpuCol.incRefCount() + case col => col + } + } + } + + new ColumnarBatch(columns, batch.numRows()) + } + } + } +} + +class RoaringBitmapIterator(val inner: PeekableLongIterator, val start: Long, val end: Long) + extends Iterator[Long] { + + override def hasNext: Boolean = { + inner.hasNext && inner.peekNext() < end + } + + override def next(): Long = { + inner.next() - start + } +} diff --git a/delta-lake/common/src/main/scala/com/nvidia/spark/rapids/delta/deltaUDFs.scala b/delta-lake/common/src/main/scala/com/nvidia/spark/rapids/delta/deltaUDFs.scala index 6b2c63407d7..9893545a4ad 100644 --- a/delta-lake/common/src/main/scala/com/nvidia/spark/rapids/delta/deltaUDFs.scala +++ b/delta-lake/common/src/main/scala/com/nvidia/spark/rapids/delta/deltaUDFs.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,19 @@ package com.nvidia.spark.rapids.delta +import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, DataOutputStream} + import ai.rapids.cudf.{ColumnVector, Scalar, Table} import ai.rapids.cudf.Table.DuplicateKeepOption import com.nvidia.spark.RapidsUDF import com.nvidia.spark.rapids.Arm.withResource +import org.roaringbitmap.longlong.Roaring64Bitmap +import org.apache.spark.sql.Encoder +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.execution.metric.SQLMetric +import org.apache.spark.sql.expressions.Aggregator +import org.apache.spark.sql.types.{BinaryType, DataType, SQLUserDefinedType, UserDefinedType} import org.apache.spark.util.AccumulatorV2 class GpuDeltaRecordTouchedFileNameUDF(accum: AccumulatorV2[String, java.util.Set[String]]) @@ -73,3 +80,77 @@ class GpuDeltaMetricUpdateUDF(metric: SQLMetric) } } } + +class GpuDeltaNoopUDF extends Function1[Boolean, Boolean] with RapidsUDF with Serializable { + override def apply(v1: Boolean): Boolean = v1 + + override def evaluateColumnar(numRows: Int, args: ColumnVector*): ColumnVector = { + require(args.length == 1) + args(0).incRefCount() + } +} + +@SQLUserDefinedType(udt = classOf[RoaringBitmapUDT]) +case class RoaringBitmapWrapper(inner: Roaring64Bitmap) { + def serializeToBytes(): Array[Byte] = { + withResource(new ByteArrayOutputStream()) { bout => + withResource(new DataOutputStream(bout)) { dao => + inner.serialize(dao) + } + bout.toByteArray + } + } +} + +object RoaringBitmapWrapper { + def deserializeFromBytes(bytes: Array[Byte]): RoaringBitmapWrapper = { + withResource(new ByteArrayInputStream(bytes)) { bin => + withResource(new DataInputStream(bin)) { din => + val ret = RoaringBitmapWrapper(new Roaring64Bitmap) + ret.inner.deserialize(din) + ret + } + } + } +} + +class RoaringBitmapUDT extends UserDefinedType[RoaringBitmapWrapper] { + + override def sqlType: DataType = BinaryType + + override def serialize(obj: RoaringBitmapWrapper): Any = { + obj.serializeToBytes() + } + + override def deserialize(datum: Any): RoaringBitmapWrapper = { + datum match { + case b: Array[Byte] => RoaringBitmapWrapper.deserializeFromBytes(b) + case t => throw new IllegalArgumentException(s"t: ${t.getClass}") + } + } + + override def userClass: Class[RoaringBitmapWrapper] = classOf[RoaringBitmapWrapper] + + override def typeName: String = "RoaringBitmap" +} + +object RoaringBitmapUDAF extends Aggregator[Long, RoaringBitmapWrapper, RoaringBitmapWrapper] { + override def zero: RoaringBitmapWrapper = RoaringBitmapWrapper(new Roaring64Bitmap()) + + override def reduce(b: RoaringBitmapWrapper, a: Long): RoaringBitmapWrapper = { + b.inner.addLong(a) + b + } + + override def merge(b1: RoaringBitmapWrapper, b2: RoaringBitmapWrapper): RoaringBitmapWrapper = { + val ret = b1.inner.clone() + ret.or(b2.inner) + RoaringBitmapWrapper(ret) + } + + override def finish(reduction: RoaringBitmapWrapper): RoaringBitmapWrapper = reduction + + override def bufferEncoder: Encoder[RoaringBitmapWrapper] = ExpressionEncoder() + + override def outputEncoder: Encoder[RoaringBitmapWrapper] = ExpressionEncoder() +} diff --git a/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/Delta24xProvider.scala b/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/Delta24xProvider.scala index d3f952b856c..f90f31300e5 100644 --- a/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/Delta24xProvider.scala +++ b/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/Delta24xProvider.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -74,7 +74,8 @@ object Delta24xProvider extends DeltaIOProvider { override def getReadFileFormat(format: FileFormat): FileFormat = { val cpuFormat = format.asInstanceOf[DeltaParquetFileFormat] - GpuDelta24xParquetFileFormat(cpuFormat.metadata, cpuFormat.isSplittable) + GpuDelta24xParquetFileFormat(cpuFormat.metadata, cpuFormat.isSplittable, + cpuFormat.disablePushDowns, cpuFormat.broadcastDvMap) } override def convertToGpu( diff --git a/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala b/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala index 709df7e9416..ef579d78e6f 100644 --- a/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala +++ b/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,18 +16,32 @@ package com.nvidia.spark.rapids.delta.delta24x -import com.nvidia.spark.rapids.delta.GpuDeltaParquetFileFormat +import java.net.URI + +import com.nvidia.spark.rapids.{GpuMetric, RapidsConf} +import com.nvidia.spark.rapids.delta.{GpuDeltaParquetFileFormat, RoaringBitmapWrapper} +import com.nvidia.spark.rapids.delta.GpuDeltaParquetFileFormatUtils.addMetadataColumnToIterator +import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path +import org.apache.spark.broadcast.Broadcast import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.delta.{DeltaColumnMappingMode, IdMapping} +import org.apache.spark.sql.delta.DeltaParquetFileFormat.DeletionVectorDescriptorWithFilterType import org.apache.spark.sql.delta.actions.Metadata +import org.apache.spark.sql.execution.datasources.PartitionedFile import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.sources.Filter import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.vectorized.ColumnarBatch case class GpuDelta24xParquetFileFormat( metadata: Metadata, - isSplittable: Boolean) extends GpuDeltaParquetFileFormat { + isSplittable: Boolean, + disablePushDown: Boolean, + broadcastDvMap: Option[Broadcast[Map[URI, DeletionVectorDescriptorWithFilterType]]]) + extends GpuDeltaParquetFileFormat { override val columnMappingMode: DeltaColumnMappingMode = metadata.columnMappingMode override val referenceSchema: StructType = metadata.schema @@ -46,6 +60,47 @@ case class GpuDelta24xParquetFileFormat( options: Map[String, String], path: Path): Boolean = isSplittable + override def buildReaderWithPartitionValuesAndMetrics( + sparkSession: SparkSession, + dataSchema: StructType, + partitionSchema: StructType, + requiredSchema: StructType, + filters: Seq[Filter], + options: Map[String, String], + hadoopConf: Configuration, + metrics: Map[String, GpuMetric], + alluxioPathReplacementMap: Option[Map[String, String]]) + : PartitionedFile => Iterator[InternalRow] = { + + + val dataReader = super.buildReaderWithPartitionValuesAndMetrics( + sparkSession, + dataSchema, + partitionSchema, + requiredSchema, + if (disablePushDown) Seq.empty else filters, + options, + hadoopConf, + metrics, + alluxioPathReplacementMap) + + val delVecs = broadcastDvMap + val maxDelVecScatterBatchSize = RapidsConf + .DELTA_LOW_SHUFFLE_MERGE_SCATTER_DEL_VECTOR_BATCH_SIZE + .get(sparkSession.sessionState.conf) + + (file: PartitionedFile) => { + val input = dataReader(file) + val dv = delVecs.flatMap(_.value.get(new URI(file.filePath.toString()))) + .map(dv => RoaringBitmapWrapper.deserializeFromBytes(dv.descriptor.inlineData).inner) + addMetadataColumnToIterator(prepareSchema(requiredSchema), + dv, + input.asInstanceOf[Iterator[ColumnarBatch]], + maxDelVecScatterBatchSize) + .asInstanceOf[Iterator[InternalRow]] + } + } + /** * We sometimes need to replace FileFormat within LogicalPlans, so we have to override * `equals` to ensure file format changes are captured diff --git a/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/MergeIntoCommandMeta.scala b/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/MergeIntoCommandMeta.scala index 4b4dfb624b5..8ce813ef011 100644 --- a/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/MergeIntoCommandMeta.scala +++ b/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/MergeIntoCommandMeta.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,14 @@ package com.nvidia.spark.rapids.delta.delta24x -import com.nvidia.spark.rapids.{DataFromReplacementRule, RapidsConf, RapidsMeta, RunnableCommandMeta} +import com.nvidia.spark.rapids.{DataFromReplacementRule, RapidsConf, RapidsMeta, RapidsReaderType, RunnableCommandMeta} import com.nvidia.spark.rapids.delta.RapidsDeltaUtils +import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession import org.apache.spark.sql.delta.commands.MergeIntoCommand import org.apache.spark.sql.delta.rapids.GpuDeltaLog -import org.apache.spark.sql.delta.rapids.delta24x.GpuMergeIntoCommand +import org.apache.spark.sql.delta.rapids.delta24x.{GpuLowShuffleMergeCommand, GpuMergeIntoCommand} import org.apache.spark.sql.execution.command.RunnableCommand class MergeIntoCommandMeta( @@ -30,12 +31,12 @@ class MergeIntoCommandMeta( conf: RapidsConf, parent: Option[RapidsMeta[_, _, _]], rule: DataFromReplacementRule) - extends RunnableCommandMeta[MergeIntoCommand](mergeCmd, conf, parent, rule) { + extends RunnableCommandMeta[MergeIntoCommand](mergeCmd, conf, parent, rule) with Logging { override def tagSelfForGpu(): Unit = { if (!conf.isDeltaWriteEnabled) { willNotWorkOnGpu("Delta Lake output acceleration has been disabled. To enable set " + - s"${RapidsConf.ENABLE_DELTA_WRITE} to true") + s"${RapidsConf.ENABLE_DELTA_WRITE} to true") } if (mergeCmd.notMatchedBySourceClauses.nonEmpty) { // https://github.com/NVIDIA/spark-rapids/issues/8415 @@ -48,14 +49,43 @@ class MergeIntoCommandMeta( } override def convertToGpu(): RunnableCommand = { - GpuMergeIntoCommand( - mergeCmd.source, - mergeCmd.target, - new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), - mergeCmd.condition, - mergeCmd.matchedClauses, - mergeCmd.notMatchedClauses, - mergeCmd.notMatchedBySourceClauses, - mergeCmd.migratedSchema)(conf) + // TODO: Currently we only support low shuffler merge only when parquet per file read is enabled + // due to the limitation of implementing row index metadata column. + if (conf.isDeltaLowShuffleMergeEnabled) { + if (conf.isParquetPerFileReadEnabled) { + GpuLowShuffleMergeCommand( + mergeCmd.source, + mergeCmd.target, + new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), + mergeCmd.condition, + mergeCmd.matchedClauses, + mergeCmd.notMatchedClauses, + mergeCmd.notMatchedBySourceClauses, + mergeCmd.migratedSchema)(conf) + } else { + logWarning(s"""Low shuffle merge disabled since ${RapidsConf.PARQUET_READER_TYPE} is + not set to ${RapidsReaderType.PERFILE}. Falling back to classic merge.""") + GpuMergeIntoCommand( + mergeCmd.source, + mergeCmd.target, + new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), + mergeCmd.condition, + mergeCmd.matchedClauses, + mergeCmd.notMatchedClauses, + mergeCmd.notMatchedBySourceClauses, + mergeCmd.migratedSchema)(conf) + } + } else { + GpuMergeIntoCommand( + mergeCmd.source, + mergeCmd.target, + new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), + mergeCmd.condition, + mergeCmd.matchedClauses, + mergeCmd.notMatchedClauses, + mergeCmd.notMatchedBySourceClauses, + mergeCmd.migratedSchema)(conf) + } } + } diff --git a/delta-lake/delta-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/GpuLowShuffleMergeCommand.scala b/delta-lake/delta-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/GpuLowShuffleMergeCommand.scala new file mode 100644 index 00000000000..9c27d28ebd3 --- /dev/null +++ b/delta-lake/delta-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/GpuLowShuffleMergeCommand.scala @@ -0,0 +1,1084 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * This file was derived from MergeIntoCommand.scala + * in the Delta Lake project at https://github.com/delta-io/delta. + * + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.rapids.delta24x + +import java.net.URI +import java.util.concurrent.TimeUnit + +import scala.collection.mutable + +import com.nvidia.spark.rapids.{GpuOverrides, RapidsConf, SparkPlanMeta} +import com.nvidia.spark.rapids.RapidsConf.DELTA_LOW_SHUFFLE_MERGE_DEL_VECTOR_BROADCAST_THRESHOLD +import com.nvidia.spark.rapids.delta._ +import com.nvidia.spark.rapids.delta.GpuDeltaParquetFileFormatUtils._ +import com.nvidia.spark.rapids.shims.FileSourceScanExecMeta +import org.roaringbitmap.longlong.Roaring64Bitmap + +import org.apache.spark.SparkContext +import org.apache.spark.internal.Logging +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.expressions.{Alias, And, Attribute, AttributeReference, CaseWhen, Expression, Literal, NamedExpression, PredicateHelper} +import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral +import org.apache.spark.sql.catalyst.plans.logical.{DeltaMergeAction, DeltaMergeIntoClause, DeltaMergeIntoMatchedClause, DeltaMergeIntoMatchedDeleteClause, DeltaMergeIntoMatchedUpdateClause, DeltaMergeIntoNotMatchedBySourceClause, DeltaMergeIntoNotMatchedBySourceDeleteClause, DeltaMergeIntoNotMatchedBySourceUpdateClause, DeltaMergeIntoNotMatchedClause, DeltaMergeIntoNotMatchedInsertClause, LogicalPlan, Project} +import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap +import org.apache.spark.sql.delta.{DeltaErrors, DeltaLog, DeltaOperations, DeltaParquetFileFormat, DeltaTableUtils, DeltaUDF, NoMapping, OptimisticTransaction, RowIndexFilterType} +import org.apache.spark.sql.delta.DeltaOperations.MergePredicate +import org.apache.spark.sql.delta.DeltaParquetFileFormat.DeletionVectorDescriptorWithFilterType +import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, DeletionVectorDescriptor, FileAction} +import org.apache.spark.sql.delta.commands.DeltaCommand +import org.apache.spark.sql.delta.rapids.{GpuDeltaLog, GpuOptimisticTransactionBase} +import org.apache.spark.sql.delta.rapids.delta24x.MergeExecutor.{toDeletionVector, totalBytesAndDistinctPartitionValues, INCR_METRICS_COL, INCR_METRICS_FIELD, ROW_DROPPED_COL, ROW_DROPPED_FIELD, SOURCE_ROW_PRESENT_COL, SOURCE_ROW_PRESENT_FIELD, TARGET_ROW_PRESENT_COL, TARGET_ROW_PRESENT_FIELD} +import org.apache.spark.sql.delta.schema.ImplicitMetadataOperation +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.util.{AnalysisHelper, DeltaFileOperations} +import org.apache.spark.sql.execution.{SparkPlan, SQLExecution} +import org.apache.spark.sql.execution.command.LeafRunnableCommand +import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} +import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.types.{BooleanType, LongType, StringType, StructField, StructType} + +/** + * GPU version of Delta Lake's low shuffle merge implementation. + * + * Performs a merge of a source query/table into a Delta table. + * + * Issues an error message when the ON search_condition of the MERGE statement can match + * a single row from the target table with multiple rows of the source table-reference. + * Different from the original implementation, it optimized writing touched unmodified target files. + * + * Algorithm: + * + * Phase 1: Find the input files in target that are touched by the rows that satisfy + * the condition and verify that no two source rows match with the same target row. + * This is implemented as an inner-join using the given condition. See [[findTouchedFiles]] + * for more details. + * + * Phase 2: Read the touched files again and write new files with updated and/or inserted rows + * without copying unmodified rows. + * + * Phase 3: Read the touched files again and write new files with unmodified rows in target table, + * trying to keep its original order and avoid shuffle as much as possible. + * + * Phase 4: Use the Delta protocol to atomically remove the touched files and add the new files. + * + * @param source Source data to merge from + * @param target Target table to merge into + * @param gpuDeltaLog Delta log to use + * @param condition Condition for a source row to match with a target row + * @param matchedClauses All info related to matched clauses. + * @param notMatchedClauses All info related to not matched clause. + * @param migratedSchema The final schema of the target - may be changed by schema evolution. + */ +case class GpuLowShuffleMergeCommand( + @transient source: LogicalPlan, + @transient target: LogicalPlan, + @transient gpuDeltaLog: GpuDeltaLog, + condition: Expression, + matchedClauses: Seq[DeltaMergeIntoMatchedClause], + notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], + notMatchedBySourceClauses: Seq[DeltaMergeIntoNotMatchedBySourceClause], + migratedSchema: Option[StructType])( + @transient val rapidsConf: RapidsConf) + extends LeafRunnableCommand + with DeltaCommand with PredicateHelper with AnalysisHelper with ImplicitMetadataOperation { + + import SQLMetrics._ + + override val otherCopyArgs: Seq[AnyRef] = Seq(rapidsConf) + + override val canMergeSchema: Boolean = conf.getConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE) + override val canOverwriteSchema: Boolean = false + + override val output: Seq[Attribute] = Seq( + AttributeReference("num_affected_rows", LongType)(), + AttributeReference("num_updated_rows", LongType)(), + AttributeReference("num_deleted_rows", LongType)(), + AttributeReference("num_inserted_rows", LongType)()) + + @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() + @transient private[delta] lazy val targetDeltaLog: DeltaLog = gpuDeltaLog.deltaLog + + override lazy val metrics = Map[String, SQLMetric]( + "numSourceRows" -> createMetric(sc, "number of source rows"), + "numSourceRowsInSecondScan" -> + createMetric(sc, "number of source rows (during repeated scan)"), + "numTargetRowsCopied" -> createMetric(sc, "number of target rows rewritten unmodified"), + "numTargetRowsInserted" -> createMetric(sc, "number of inserted rows"), + "numTargetRowsUpdated" -> createMetric(sc, "number of updated rows"), + "numTargetRowsDeleted" -> createMetric(sc, "number of deleted rows"), + "numTargetRowsMatchedUpdated" -> createMetric(sc, "number of target rows updated when matched"), + "numTargetRowsMatchedDeleted" -> createMetric(sc, "number of target rows deleted when matched"), + "numTargetRowsNotMatchedBySourceUpdated" -> createMetric(sc, + "number of target rows updated when not matched by source"), + "numTargetRowsNotMatchedBySourceDeleted" -> createMetric(sc, + "number of target rows deleted when not matched by source"), + "numTargetFilesBeforeSkipping" -> createMetric(sc, "number of target files before skipping"), + "numTargetFilesAfterSkipping" -> createMetric(sc, "number of target files after skipping"), + "numTargetFilesRemoved" -> createMetric(sc, "number of files removed to target"), + "numTargetFilesAdded" -> createMetric(sc, "number of files added to target"), + "numTargetChangeFilesAdded" -> + createMetric(sc, "number of change data capture files generated"), + "numTargetChangeFileBytes" -> + createMetric(sc, "total size of change data capture files generated"), + "numTargetBytesBeforeSkipping" -> createMetric(sc, "number of target bytes before skipping"), + "numTargetBytesAfterSkipping" -> createMetric(sc, "number of target bytes after skipping"), + "numTargetBytesRemoved" -> createMetric(sc, "number of target bytes removed"), + "numTargetBytesAdded" -> createMetric(sc, "number of target bytes added"), + "numTargetPartitionsAfterSkipping" -> + createMetric(sc, "number of target partitions after skipping"), + "numTargetPartitionsRemovedFrom" -> + createMetric(sc, "number of target partitions from which files were removed"), + "numTargetPartitionsAddedTo" -> + createMetric(sc, "number of target partitions to which files were added"), + "executionTimeMs" -> + createMetric(sc, "time taken to execute the entire operation"), + "scanTimeMs" -> + createMetric(sc, "time taken to scan the files for matches"), + "rewriteTimeMs" -> + createMetric(sc, "time taken to rewrite the matched files")) + + /** Whether this merge statement has only a single insert (NOT MATCHED) clause. */ + protected def isSingleInsertOnly: Boolean = matchedClauses.isEmpty && + notMatchedClauses.length == 1 + + override def run(spark: SparkSession): Seq[Row] = { + recordDeltaOperation(targetDeltaLog, "delta.dml.lowshufflemerge") { + val startTime = System.nanoTime() + val result = gpuDeltaLog.withNewTransaction { deltaTxn => + if (target.schema.size != deltaTxn.metadata.schema.size) { + throw DeltaErrors.schemaChangedSinceAnalysis( + atAnalysis = target.schema, latestSchema = deltaTxn.metadata.schema) + } + + if (canMergeSchema) { + updateMetadata( + spark, deltaTxn, migratedSchema.getOrElse(target.schema), + deltaTxn.metadata.partitionColumns, deltaTxn.metadata.configuration, + isOverwriteMode = false, rearrangeOnly = false) + } + + + val (executor, fallback) = { + val context = MergeExecutorContext(this, spark, deltaTxn, rapidsConf) + if (isSingleInsertOnly && spark.conf.get(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED)) { + (new InsertOnlyMergeExecutor(context), false) + } else { + val executor = new LowShuffleMergeExecutor(context) + (executor, executor.shouldFallback()) + } + } + + if (fallback) { + None + } else { + Some(runLowShuffleMerge(spark, startTime, deltaTxn, executor)) + } + } + + result match { + case Some(row) => row + case None => + // We should rollback to normal gpu + new GpuMergeIntoCommand(source, target, gpuDeltaLog, condition, matchedClauses, + notMatchedClauses, notMatchedBySourceClauses, migratedSchema)(rapidsConf) + .run(spark) + } + } + } + + + private def runLowShuffleMerge( + spark: SparkSession, + startTime: Long, + deltaTxn: GpuOptimisticTransactionBase, + mergeExecutor: MergeExecutor): Seq[Row] = { + val deltaActions = mergeExecutor.execute() + // Metrics should be recorded before commit (where they are written to delta logs). + metrics("executionTimeMs").set((System.nanoTime() - startTime) / 1000 / 1000) + deltaTxn.registerSQLMetrics(spark, metrics) + + // This is a best-effort sanity check. + if (metrics("numSourceRowsInSecondScan").value >= 0 && + metrics("numSourceRows").value != metrics("numSourceRowsInSecondScan").value) { + log.warn(s"Merge source has ${metrics("numSourceRows").value} rows in initial scan but " + + s"${metrics("numSourceRowsInSecondScan").value} rows in second scan") + if (conf.getConf(DeltaSQLConf.MERGE_FAIL_IF_SOURCE_CHANGED)) { + throw DeltaErrors.sourceNotDeterministicInMergeException(spark) + } + } + + deltaTxn.commit( + deltaActions, + DeltaOperations.Merge( + Option(condition), + matchedClauses.map(DeltaOperations.MergePredicate(_)), + notMatchedClauses.map(DeltaOperations.MergePredicate(_)), + // We do not support notMatchedBySourcePredicates yet and fall back to CPU + // See https://github.com/NVIDIA/spark-rapids/issues/8415 + notMatchedBySourcePredicates = Seq.empty[MergePredicate] + )) + + // Record metrics + val stats = GpuMergeStats.fromMergeSQLMetrics( + metrics, + condition, + matchedClauses, + notMatchedClauses, + notMatchedBySourceClauses, + deltaTxn.metadata.partitionColumns.nonEmpty) + recordDeltaEvent(targetDeltaLog, "delta.dml.merge.stats", data = stats) + + + spark.sharedState.cacheManager.recacheByPlan(spark, target) + + // This is needed to make the SQL metrics visible in the Spark UI. Also this needs + // to be outside the recordMergeOperation because this method will update some metric. + val executionId = spark.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) + SQLMetrics.postDriverMetricUpdates(spark.sparkContext, executionId, metrics.values.toSeq) + Seq(Row(metrics("numTargetRowsUpdated").value + metrics("numTargetRowsDeleted").value + + metrics("numTargetRowsInserted").value, metrics("numTargetRowsUpdated").value, + metrics("numTargetRowsDeleted").value, metrics("numTargetRowsInserted").value)) + } + + /** + * Execute the given `thunk` and return its result while recording the time taken to do it. + * + * @param sqlMetricName name of SQL metric to update with the time taken by the thunk + * @param thunk the code to execute + */ + private[delta] def recordMergeOperation[A](sqlMetricName: String)(thunk: => A): A = { + val startTimeNs = System.nanoTime() + val r = thunk + val timeTakenMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs) + if (sqlMetricName != null && timeTakenMs > 0) { + metrics(sqlMetricName) += timeTakenMs + } + r + } + + /** Expressions to increment SQL metrics */ + private[delta] def makeMetricUpdateUDF(name: String, deterministic: Boolean = false) + : Expression = { + // only capture the needed metric in a local variable + val metric = metrics(name) + var u = DeltaUDF.boolean(new GpuDeltaMetricUpdateUDF(metric)) + if (!deterministic) { + u = u.asNondeterministic() + } + u.apply().expr + } +} + +/** + * Context merge execution. + */ +case class MergeExecutorContext(cmd: GpuLowShuffleMergeCommand, + spark: SparkSession, + deltaTxn: OptimisticTransaction, + rapidsConf: RapidsConf) + +trait MergeExecutor extends AnalysisHelper with PredicateHelper with Logging { + + val context: MergeExecutorContext + + + /** + * Map to get target output attributes by name. + * The case sensitivity of the map is set accordingly to Spark configuration. + */ + @transient private lazy val targetOutputAttributesMap: Map[String, Attribute] = { + val attrMap: Map[String, Attribute] = context.cmd.target + .outputSet.view + .map(attr => attr.name -> attr).toMap + if (context.cmd.conf.caseSensitiveAnalysis) { + attrMap + } else { + CaseInsensitiveMap(attrMap) + } + } + + def execute(): Seq[FileAction] + + protected def targetOutputCols: Seq[NamedExpression] = { + context.deltaTxn.metadata.schema.map { col => + targetOutputAttributesMap + .get(col.name) + .map { a => + AttributeReference(col.name, col.dataType, col.nullable)(a.exprId) + } + .getOrElse(Alias(Literal(null), col.name)()) + } + } + + /** + * Build a DataFrame using the given `files` that has the same output columns (exprIds) + * as the `target` logical plan, so that existing update/insert expressions can be applied + * on this new plan. + */ + protected def buildTargetDFWithFiles(files: Seq[AddFile]): DataFrame = { + val targetOutputColsMap = { + val colsMap: Map[String, NamedExpression] = targetOutputCols.view + .map(col => col.name -> col).toMap + if (context.cmd.conf.caseSensitiveAnalysis) { + colsMap + } else { + CaseInsensitiveMap(colsMap) + } + } + + val plan = { + // We have to do surgery to use the attributes from `targetOutputCols` to scan the table. + // In cases of schema evolution, they may not be the same type as the original attributes. + val original = + context.deltaTxn.deltaLog.createDataFrame(context.deltaTxn.snapshot, files) + .queryExecution + .analyzed + val transformed = original.transform { + case LogicalRelation(base, _, catalogTbl, isStreaming) => + LogicalRelation( + base, + // We can ignore the new columns which aren't yet AttributeReferences. + targetOutputCols.collect { case a: AttributeReference => a }, + catalogTbl, + isStreaming) + } + + // In case of schema evolution & column mapping, we would also need to rebuild the file + // format because under column mapping, the reference schema within DeltaParquetFileFormat + // that is used to populate metadata needs to be updated + if (context.deltaTxn.metadata.columnMappingMode != NoMapping) { + val updatedFileFormat = context.deltaTxn.deltaLog.fileFormat( + context.deltaTxn.deltaLog.unsafeVolatileSnapshot.protocol, context.deltaTxn.metadata) + DeltaTableUtils.replaceFileFormat(transformed, updatedFileFormat) + } else { + transformed + } + } + + // For each plan output column, find the corresponding target output column (by name) and + // create an alias + val aliases = plan.output.map { + case newAttrib: AttributeReference => + val existingTargetAttrib = targetOutputColsMap.getOrElse(newAttrib.name, + throw new AnalysisException( + s"Could not find ${newAttrib.name} among the existing target output " + + targetOutputCols.mkString(","))).asInstanceOf[AttributeReference] + + if (existingTargetAttrib.exprId == newAttrib.exprId) { + // It's not valid to alias an expression to its own exprId (this is considered a + // non-unique exprId by the analyzer), so we just use the attribute directly. + newAttrib + } else { + Alias(newAttrib, existingTargetAttrib.name)(exprId = existingTargetAttrib.exprId) + } + } + + Dataset.ofRows(context.spark, Project(aliases, plan)) + } + + + /** + * Repartitions the output DataFrame by the partition columns if table is partitioned + * and `merge.repartitionBeforeWrite.enabled` is set to true. + */ + protected def repartitionIfNeeded(df: DataFrame): DataFrame = { + val partitionColumns = context.deltaTxn.metadata.partitionColumns + // TODO: We should remove this method and use optimized write instead, see + // https://github.com/NVIDIA/spark-rapids/issues/10417 + if (partitionColumns.nonEmpty && context.spark.conf.get(DeltaSQLConf + .MERGE_REPARTITION_BEFORE_WRITE)) { + df.repartition(partitionColumns.map(col): _*) + } else { + df + } + } + + protected def sourceDF: DataFrame = { + // UDF to increment metrics + val incrSourceRowCountExpr = context.cmd.makeMetricUpdateUDF("numSourceRows") + Dataset.ofRows(context.spark, context.cmd.source) + .filter(new Column(incrSourceRowCountExpr)) + } + + /** Whether this merge statement has no insert (NOT MATCHED) clause. */ + protected def hasNoInserts: Boolean = context.cmd.notMatchedClauses.isEmpty + + +} + +/** + * This is an optimization of the case when there is no update clause for the merge. + * We perform an left anti join on the source data to find the rows to be inserted. + * + * This will currently only optimize for the case when there is a _single_ notMatchedClause. + */ +class InsertOnlyMergeExecutor(override val context: MergeExecutorContext) extends MergeExecutor { + override def execute(): Seq[FileAction] = { + context.cmd.recordMergeOperation(sqlMetricName = "rewriteTimeMs") { + + // UDFs to update metrics + val incrSourceRowCountExpr = context.cmd.makeMetricUpdateUDF("numSourceRows") + val incrInsertedCountExpr = context.cmd.makeMetricUpdateUDF("numTargetRowsInserted") + + val outputColNames = targetOutputCols.map(_.name) + // we use head here since we know there is only a single notMatchedClause + val outputExprs = context.cmd.notMatchedClauses.head.resolvedActions.map(_.expr) + val outputCols = outputExprs.zip(outputColNames).map { case (expr, name) => + new Column(Alias(expr, name)()) + } + + // source DataFrame + val sourceDF = Dataset.ofRows(context.spark, context.cmd.source) + .filter(new Column(incrSourceRowCountExpr)) + .filter(new Column(context.cmd.notMatchedClauses.head.condition + .getOrElse(Literal.TrueLiteral))) + + // Skip data based on the merge condition + val conjunctivePredicates = splitConjunctivePredicates(context.cmd.condition) + val targetOnlyPredicates = + conjunctivePredicates.filter(_.references.subsetOf(context.cmd.target.outputSet)) + val dataSkippedFiles = context.deltaTxn.filterFiles(targetOnlyPredicates) + + // target DataFrame + val targetDF = buildTargetDFWithFiles(dataSkippedFiles) + + val insertDf = sourceDF.join(targetDF, new Column(context.cmd.condition), "leftanti") + .select(outputCols: _*) + .filter(new Column(incrInsertedCountExpr)) + + val newFiles = context.deltaTxn + .writeFiles(repartitionIfNeeded(insertDf, + )) + + // Update metrics + context.cmd.metrics("numTargetFilesBeforeSkipping") += context.deltaTxn.snapshot.numOfFiles + context.cmd.metrics("numTargetBytesBeforeSkipping") += context.deltaTxn.snapshot.sizeInBytes + val (afterSkippingBytes, afterSkippingPartitions) = + totalBytesAndDistinctPartitionValues(dataSkippedFiles) + context.cmd.metrics("numTargetFilesAfterSkipping") += dataSkippedFiles.size + context.cmd.metrics("numTargetBytesAfterSkipping") += afterSkippingBytes + context.cmd.metrics("numTargetPartitionsAfterSkipping") += afterSkippingPartitions + context.cmd.metrics("numTargetFilesRemoved") += 0 + context.cmd.metrics("numTargetBytesRemoved") += 0 + context.cmd.metrics("numTargetPartitionsRemovedFrom") += 0 + val (addedBytes, addedPartitions) = totalBytesAndDistinctPartitionValues(newFiles) + context.cmd.metrics("numTargetFilesAdded") += newFiles.count(_.isInstanceOf[AddFile]) + context.cmd.metrics("numTargetBytesAdded") += addedBytes + context.cmd.metrics("numTargetPartitionsAddedTo") += addedPartitions + newFiles + } + } +} + + +/** + * This is an optimized algorithm for merge statement, where we avoid shuffling the unmodified + * target data. + * + * The algorithm is as follows: + * 1. Find touched target files in the target table by joining the source and target data, with + * collecting joined row identifiers as (`__metadata_file_path`, `__metadata_row_idx`) pairs. + * 2. Read the touched files again and write new files with updated and/or inserted rows + * without coping unmodified data from target table, but filtering target table with collected + * rows mentioned above. + * 3. Read the touched files again, filtering unmodified rows with collected row identifiers + * collected in first step, and saving them without shuffle. + */ +class LowShuffleMergeExecutor(override val context: MergeExecutorContext) extends MergeExecutor { + + // We over-count numTargetRowsDeleted when there are multiple matches; + // this is the amount of the overcount, so we can subtract it to get a correct final metric. + private var multipleMatchDeleteOnlyOvercount: Option[Long] = None + + // UDFs to update metrics + private val incrSourceRowCountExpr: Expression = context.cmd. + makeMetricUpdateUDF("numSourceRowsInSecondScan") + private val incrUpdatedCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsUpdated") + private val incrUpdatedMatchedCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsMatchedUpdated") + private val incrUpdatedNotMatchedBySourceCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsNotMatchedBySourceUpdated") + private val incrInsertedCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsInserted") + private val incrDeletedCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsDeleted") + private val incrDeletedMatchedCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsMatchedDeleted") + private val incrDeletedNotMatchedBySourceCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsNotMatchedBySourceDeleted") + + private def updateOutput(resolvedActions: Seq[DeltaMergeAction], incrExpr: Expression) + : Seq[Expression] = { + resolvedActions.map(_.expr) :+ + Literal.FalseLiteral :+ + UnresolvedAttribute(TARGET_ROW_PRESENT_COL) :+ + UnresolvedAttribute(SOURCE_ROW_PRESENT_COL) :+ + incrExpr + } + + private def deleteOutput(incrExpr: Expression): Seq[Expression] = { + targetOutputCols :+ + TrueLiteral :+ + UnresolvedAttribute(TARGET_ROW_PRESENT_COL) :+ + UnresolvedAttribute(SOURCE_ROW_PRESENT_COL) :+ + incrExpr + } + + private def insertOutput(resolvedActions: Seq[DeltaMergeAction], incrExpr: Expression) + : Seq[Expression] = { + resolvedActions.map(_.expr) :+ + Literal.FalseLiteral :+ + UnresolvedAttribute(TARGET_ROW_PRESENT_COL) :+ + UnresolvedAttribute(SOURCE_ROW_PRESENT_COL) :+ + incrExpr + } + + private def clauseOutput(clause: DeltaMergeIntoClause): Seq[Expression] = clause match { + case u: DeltaMergeIntoMatchedUpdateClause => + updateOutput(u.resolvedActions, And(incrUpdatedCountExpr, incrUpdatedMatchedCountExpr)) + case _: DeltaMergeIntoMatchedDeleteClause => + deleteOutput(And(incrDeletedCountExpr, incrDeletedMatchedCountExpr)) + case i: DeltaMergeIntoNotMatchedInsertClause => + insertOutput(i.resolvedActions, incrInsertedCountExpr) + case u: DeltaMergeIntoNotMatchedBySourceUpdateClause => + updateOutput(u.resolvedActions, + And(incrUpdatedCountExpr, incrUpdatedNotMatchedBySourceCountExpr)) + case _: DeltaMergeIntoNotMatchedBySourceDeleteClause => + deleteOutput(And(incrDeletedCountExpr, incrDeletedNotMatchedBySourceCountExpr)) + } + + private def clauseCondition(clause: DeltaMergeIntoClause): Expression = { + // if condition is None, then expression always evaluates to true + clause.condition.getOrElse(TrueLiteral) + } + + /** + * Though low shuffle merge algorithm performs better than traditional merge algorithm in some + * cases, there are some case we should fallback to traditional merge executor: + * + * 1. Low shuffle merge algorithm requires generating metadata columns such as + * [[METADATA_ROW_IDX_COL]], [[METADATA_ROW_DEL_COL]], which only implemented on + * [[org.apache.spark.sql.rapids.GpuFileSourceScanExec]]. That means we need to fallback to + * this normal executor when [[org.apache.spark.sql.rapids.GpuFileSourceScanExec]] is disabled + * for some reason. + * 2. Low shuffle merge algorithm currently needs to broadcast deletion vector, which may + * introduce extra overhead. It maybe better to fallback to this algorithm when the changeset + * it too large. + */ + private[delta] def shouldFallback(): Boolean = { + // Trying to detect if we can execute finding touched files. + val touchFilePlanOverrideSucceed = verifyGpuPlan(planForFindingTouchedFiles()) { planMeta => + def check(meta: SparkPlanMeta[SparkPlan]): Boolean = { + meta match { + case scan if scan.isInstanceOf[FileSourceScanExecMeta] => scan + .asInstanceOf[FileSourceScanExecMeta] + .wrapped + .schema + .fieldNames + .contains(METADATA_ROW_IDX_COL) && scan.canThisBeReplaced + case m => m.childPlans.exists(check) + } + } + + check(planMeta) + } + if (!touchFilePlanOverrideSucceed) { + logWarning("Unable to override file scan for low shuffle merge for finding touched files " + + "plan, fallback to tradition merge.") + return true + } + + // Trying to detect if we can execute the merge plan. + val mergePlanOverrideSucceed = verifyGpuPlan(planForMergeExecution(touchedFiles)) { planMeta => + var overrideCount = 0 + def count(meta: SparkPlanMeta[SparkPlan]): Unit = { + meta match { + case scan if scan.isInstanceOf[FileSourceScanExecMeta] => + if (scan.asInstanceOf[FileSourceScanExecMeta] + .wrapped.schema.fieldNames.contains(METADATA_ROW_DEL_COL) && scan.canThisBeReplaced) { + overrideCount += 1 + } + case m => m.childPlans.foreach(count) + } + } + + count(planMeta) + overrideCount == 2 + } + + if (!mergePlanOverrideSucceed) { + logWarning("Unable to override file scan for low shuffle merge for merge plan, fallback to " + + "tradition merge.") + return true + } + + val deletionVectorSize = touchedFiles.values.map(_._1.serializedSizeInBytes()).sum + val maxDelVectorSize = context.rapidsConf + .get(DELTA_LOW_SHUFFLE_MERGE_DEL_VECTOR_BROADCAST_THRESHOLD) + if (deletionVectorSize > maxDelVectorSize) { + logWarning( + s"""Low shuffle merge can't be executed because broadcast deletion vector count + |$deletionVectorSize is large than max value $maxDelVectorSize """.stripMargin) + return true + } + + false + } + + private def verifyGpuPlan(input: DataFrame)(checkPlanMeta: SparkPlanMeta[SparkPlan] => Boolean) + : Boolean = { + val overridePlan = GpuOverrides.wrapAndTagPlan(input.queryExecution.sparkPlan, + context.rapidsConf) + checkPlanMeta(overridePlan) + } + + override def execute(): Seq[FileAction] = { + val newFiles = context.cmd.withStatusCode("DELTA", + s"Rewriting ${touchedFiles.size} files and saving modified data") { + val df = planForMergeExecution(touchedFiles) + context.deltaTxn.writeFiles(df) + } + + // Update metrics + val (addedBytes, addedPartitions) = totalBytesAndDistinctPartitionValues(newFiles) + context.cmd.metrics("numTargetFilesAdded") += newFiles.count(_.isInstanceOf[AddFile]) + context.cmd.metrics("numTargetChangeFilesAdded") += newFiles.count(_.isInstanceOf[AddCDCFile]) + context.cmd.metrics("numTargetChangeFileBytes") += newFiles.collect { + case f: AddCDCFile => f.size + } + .sum + context.cmd.metrics("numTargetBytesAdded") += addedBytes + context.cmd.metrics("numTargetPartitionsAddedTo") += addedPartitions + + if (multipleMatchDeleteOnlyOvercount.isDefined) { + // Compensate for counting duplicates during the query. + val actualRowsDeleted = + context.cmd.metrics("numTargetRowsDeleted").value - multipleMatchDeleteOnlyOvercount.get + assert(actualRowsDeleted >= 0) + context.cmd.metrics("numTargetRowsDeleted").set(actualRowsDeleted) + } + + touchedFiles.values.map(_._2).map(_.remove).toSeq ++ newFiles + } + + private lazy val dataSkippedFiles: Seq[AddFile] = { + // Skip data based on the merge condition + val targetOnlyPredicates = splitConjunctivePredicates(context.cmd.condition) + .filter(_.references.subsetOf(context.cmd.target.outputSet)) + context.deltaTxn.filterFiles(targetOnlyPredicates) + } + + private lazy val dataSkippedTargetDF: DataFrame = { + addRowIndexMetaColumn(buildTargetDFWithFiles(dataSkippedFiles)) + } + + private lazy val touchedFiles: Map[String, (Roaring64Bitmap, AddFile)] = this.findTouchedFiles() + + private def planForFindingTouchedFiles(): DataFrame = { + + // Apply inner join to between source and target using the merge condition to find matches + // In addition, we attach two columns + // - METADATA_ROW_IDX column to identify target row in file + // - FILE_PATH_COL the target file name the row is from to later identify the files touched + // by matched rows + val targetDF = dataSkippedTargetDF.withColumn(FILE_PATH_COL, input_file_name()) + + sourceDF.join(targetDF, new Column(context.cmd.condition), "inner") + } + + private def planForMergeExecution(touchedFiles: Map[String, (Roaring64Bitmap, AddFile)]) + : DataFrame = { + getModifiedDF(touchedFiles).unionAll(getUnmodifiedDF(touchedFiles)) + } + + /** + * Find the target table files that contain the rows that satisfy the merge condition. This is + * implemented as an inner-join between the source query/table and the target table using + * the merge condition. + */ + private def findTouchedFiles(): Map[String, (Roaring64Bitmap, AddFile)] = + context.cmd.recordMergeOperation(sqlMetricName = "scanTimeMs") { + context.spark.udf.register("row_index_set", udaf(RoaringBitmapUDAF)) + // Process the matches from the inner join to record touched files and find multiple matches + val collectTouchedFiles = planForFindingTouchedFiles() + .select(col(FILE_PATH_COL), col(METADATA_ROW_IDX_COL)) + .groupBy(FILE_PATH_COL) + .agg( + expr(s"row_index_set($METADATA_ROW_IDX_COL) as row_idxes"), + count("*").as("count")) + .collect().map(row => { + val filename = row.getAs[String](FILE_PATH_COL) + val rowIdxSet = row.getAs[RoaringBitmapWrapper]("row_idxes").inner + val count = row.getAs[Long]("count") + (filename, (rowIdxSet, count)) + }) + .toMap + + val duplicateCount = { + val distinctMatchedRowCounts = collectTouchedFiles.values + .map(_._1.getLongCardinality).sum + val allMatchedRowCounts = collectTouchedFiles.values.map(_._2).sum + allMatchedRowCounts - distinctMatchedRowCounts + } + + val hasMultipleMatches = duplicateCount > 0 + + // Throw error if multiple matches are ambiguous or cannot be computed correctly. + val canBeComputedUnambiguously = { + // Multiple matches are not ambiguous when there is only one unconditional delete as + // all the matched row pairs in the 2nd join in `writeAllChanges` will get deleted. + val isUnconditionalDelete = context.cmd.matchedClauses.headOption match { + case Some(DeltaMergeIntoMatchedDeleteClause(None)) => true + case _ => false + } + context.cmd.matchedClauses.size == 1 && isUnconditionalDelete + } + + if (hasMultipleMatches && !canBeComputedUnambiguously) { + throw DeltaErrors.multipleSourceRowMatchingTargetRowInMergeException(context.spark) + } + + if (hasMultipleMatches) { + // This is only allowed for delete-only queries. + // This query will count the duplicates for numTargetRowsDeleted in Job 2, + // because we count matches after the join and not just the target rows. + // We have to compensate for this by subtracting the duplicates later, + // so we need to record them here. + multipleMatchDeleteOnlyOvercount = Some(duplicateCount) + } + + // Get the AddFiles using the touched file names. + val touchedFileNames = collectTouchedFiles.keys.toSeq + + val nameToAddFileMap = context.cmd.generateCandidateFileMap( + context.cmd.targetDeltaLog.dataPath, + dataSkippedFiles) + + val touchedAddFiles = touchedFileNames.map(f => + context.cmd.getTouchedFile(context.cmd.targetDeltaLog.dataPath, f, nameToAddFileMap)) + .map(f => (DeltaFileOperations + .absolutePath(context.cmd.targetDeltaLog.dataPath.toString, f.path) + .toString, f)).toMap + + // When the target table is empty, and the optimizer optimized away the join entirely + // numSourceRows will be incorrectly 0. + // We need to scan the source table once to get the correct + // metric here. + if (context.cmd.metrics("numSourceRows").value == 0 && + (dataSkippedFiles.isEmpty || dataSkippedTargetDF.take(1).isEmpty)) { + val numSourceRows = sourceDF.count() + context.cmd.metrics("numSourceRows").set(numSourceRows) + } + + // Update metrics + context.cmd.metrics("numTargetFilesBeforeSkipping") += context.deltaTxn.snapshot.numOfFiles + context.cmd.metrics("numTargetBytesBeforeSkipping") += context.deltaTxn.snapshot.sizeInBytes + val (afterSkippingBytes, afterSkippingPartitions) = + totalBytesAndDistinctPartitionValues(dataSkippedFiles) + context.cmd.metrics("numTargetFilesAfterSkipping") += dataSkippedFiles.size + context.cmd.metrics("numTargetBytesAfterSkipping") += afterSkippingBytes + context.cmd.metrics("numTargetPartitionsAfterSkipping") += afterSkippingPartitions + val (removedBytes, removedPartitions) = + totalBytesAndDistinctPartitionValues(touchedAddFiles.values.toSeq) + context.cmd.metrics("numTargetFilesRemoved") += touchedAddFiles.size + context.cmd.metrics("numTargetBytesRemoved") += removedBytes + context.cmd.metrics("numTargetPartitionsRemovedFrom") += removedPartitions + + collectTouchedFiles.map(kv => (kv._1, (kv._2._1, touchedAddFiles(kv._1)))) + } + + + /** + * Modify original data frame to insert + * [[GpuDeltaParquetFileFormatUtils.METADATA_ROW_IDX_COL]]. + */ + private def addRowIndexMetaColumn(baseDF: DataFrame): DataFrame = { + val rowIdxAttr = AttributeReference( + METADATA_ROW_IDX_COL, + METADATA_ROW_IDX_FIELD.dataType, + METADATA_ROW_IDX_FIELD.nullable)() + + val newPlan = baseDF.queryExecution.analyzed.transformUp { + case r@LogicalRelation(fs: HadoopFsRelation, _, _, _) => + val newSchema = StructType(fs.dataSchema.fields).add(METADATA_ROW_IDX_FIELD) + + // This is required to ensure that row index is correctly calculated. + val newFileFormat = fs.fileFormat.asInstanceOf[DeltaParquetFileFormat] + .copy(isSplittable = false, disablePushDowns = true) + + val newFs = fs.copy(dataSchema = newSchema, fileFormat = newFileFormat)(context.spark) + + val newOutput = r.output :+ rowIdxAttr + r.copy(relation = newFs, output = newOutput) + case p@Project(projectList, _) => + val newProjectList = projectList :+ rowIdxAttr + p.copy(projectList = newProjectList) + } + + Dataset.ofRows(context.spark, newPlan) + } + + /** + * The result is scanning target table with touched files, and added an extra + * [[METADATA_ROW_DEL_COL]] to indicate whether filtered by joining with source table in first + * step. + */ + private def getTouchedTargetDF(touchedFiles: Map[String, (Roaring64Bitmap, AddFile)]) + : DataFrame = { + // Generate a new target dataframe that has same output attributes exprIds as the target plan. + // This allows us to apply the existing resolved update/insert expressions. + val baseTargetDF = buildTargetDFWithFiles(touchedFiles.values.map(_._2).toSeq) + + val newPlan = { + val rowDelAttr = AttributeReference( + METADATA_ROW_DEL_COL, + METADATA_ROW_DEL_FIELD.dataType, + METADATA_ROW_DEL_FIELD.nullable)() + + baseTargetDF.queryExecution.analyzed.transformUp { + case r@LogicalRelation(fs: HadoopFsRelation, _, _, _) => + val newSchema = StructType(fs.dataSchema.fields).add(METADATA_ROW_DEL_FIELD) + + // This is required to ensure that row index is correctly calculated. + val newFileFormat = { + val oldFormat = fs.fileFormat.asInstanceOf[DeltaParquetFileFormat] + val dvs = touchedFiles.map(kv => (new URI(kv._1), + DeletionVectorDescriptorWithFilterType(toDeletionVector(kv._2._1), + RowIndexFilterType.UNKNOWN))) + val broadcastDVs = context.spark.sparkContext.broadcast(dvs) + + oldFormat.copy(isSplittable = false, + broadcastDvMap = Some(broadcastDVs), + disablePushDowns = true) + } + + val newFs = fs.copy(dataSchema = newSchema, fileFormat = newFileFormat)(context.spark) + + val newOutput = r.output :+ rowDelAttr + r.copy(relation = newFs, output = newOutput) + case p@Project(projectList, _) => + val newProjectList = projectList :+ rowDelAttr + p.copy(projectList = newProjectList) + } + } + + val df = Dataset.ofRows(context.spark, newPlan) + .withColumn(TARGET_ROW_PRESENT_COL, lit(true)) + + df + } + + /** + * Generate a plan by calculating modified rows. It's computed by joining source and target + * tables, where target table has been filtered by (`__metadata_file_name`, + * `__metadata_row_idx`) pairs collected in first step. + * + * Schema of `modifiedDF`: + * + * targetSchema + ROW_DROPPED_COL + TARGET_ROW_PRESENT_COL + + * SOURCE_ROW_PRESENT_COL + INCR_METRICS_COL + * INCR_METRICS_COL + * + * It consists of several parts: + * + * 1. Unmatched source rows which are inserted + * 2. Unmatched source rows which are deleted + * 3. Target rows which are updated + * 4. Target rows which are deleted + */ + private def getModifiedDF(touchedFiles: Map[String, (Roaring64Bitmap, AddFile)]): DataFrame = { + val sourceDF = this.sourceDF + .withColumn(SOURCE_ROW_PRESENT_COL, new Column(incrSourceRowCountExpr)) + + val targetDF = getTouchedTargetDF(touchedFiles) + + val joinedDF = { + val joinType = if (hasNoInserts && + context.spark.conf.get(DeltaSQLConf.MERGE_MATCHED_ONLY_ENABLED)) { + "inner" + } else { + "leftOuter" + } + val matchedTargetDF = targetDF.filter(METADATA_ROW_DEL_COL) + .drop(METADATA_ROW_DEL_COL) + + sourceDF.join(matchedTargetDF, new Column(context.cmd.condition), joinType) + } + + val modifiedRowsSchema = context.deltaTxn.metadata.schema + .add(ROW_DROPPED_FIELD) + .add(TARGET_ROW_PRESENT_FIELD.copy(nullable = true)) + .add(SOURCE_ROW_PRESENT_FIELD.copy(nullable = true)) + .add(INCR_METRICS_FIELD) + + // Here we generate a case when statement to handle all cases: + // CASE + // WHEN + // CASE WHEN + // + // WHEN + // + // ELSE + // + // WHEN + // CASE WHEN + // + // WHEN + // + // ELSE + // + // END + + val notMatchedConditions = context.cmd.notMatchedClauses.map(clauseCondition) + val notMatchedExpr = { + val deletedNotMatchedRow = { + targetOutputCols :+ + Literal.TrueLiteral :+ + Literal.FalseLiteral :+ + Literal(null) :+ + Literal.TrueLiteral + } + if (context.cmd.notMatchedClauses.isEmpty) { + // If there no `WHEN NOT MATCHED` clause, we should just delete not matched row + deletedNotMatchedRow + } else { + val notMatchedOutputs = context.cmd.notMatchedClauses.map(clauseOutput) + modifiedRowsSchema.zipWithIndex.map { + case (_, idx) => + CaseWhen(notMatchedConditions.zip(notMatchedOutputs.map(_(idx))), + deletedNotMatchedRow(idx)) + } + } + } + + val matchedConditions = context.cmd.matchedClauses.map(clauseCondition) + val matchedOutputs = context.cmd.matchedClauses.map(clauseOutput) + val matchedExprs = { + val notMatchedRow = { + targetOutputCols :+ + Literal.FalseLiteral :+ + Literal.TrueLiteral :+ + Literal(null) :+ + Literal.TrueLiteral + } + if (context.cmd.matchedClauses.isEmpty) { + // If there is not matched clause, this is insert only, we should delete this row. + notMatchedRow + } else { + modifiedRowsSchema.zipWithIndex.map { + case (_, idx) => + CaseWhen(matchedConditions.zip(matchedOutputs.map(_(idx))), + notMatchedRow(idx)) + } + } + } + + val sourceRowHasNoMatch = col(TARGET_ROW_PRESENT_COL).isNull.expr + + val modifiedCols = modifiedRowsSchema.zipWithIndex.map { case (col, idx) => + val caseWhen = CaseWhen( + Seq(sourceRowHasNoMatch -> notMatchedExpr(idx)), + matchedExprs(idx)) + Column(Alias(caseWhen, col.name)()) + } + + val modifiedDF = { + + // Make this a udf to avoid catalyst to be too aggressive to even remove the join! + val noopRowDroppedCol = udf(new GpuDeltaNoopUDF()).apply(!col(ROW_DROPPED_COL)) + + val modifiedDF = joinedDF.select(modifiedCols: _*) + // This will not filter anything since they always return true, but we need to avoid + // catalyst from optimizing these udf + .filter(noopRowDroppedCol && col(INCR_METRICS_COL)) + .drop(ROW_DROPPED_COL, INCR_METRICS_COL, TARGET_ROW_PRESENT_COL, SOURCE_ROW_PRESENT_COL) + + repartitionIfNeeded(modifiedDF) + } + + modifiedDF + } + + private def getUnmodifiedDF(touchedFiles: Map[String, (Roaring64Bitmap, AddFile)]): DataFrame = { + getTouchedTargetDF(touchedFiles) + .filter(!col(METADATA_ROW_DEL_COL)) + .drop(TARGET_ROW_PRESENT_COL, METADATA_ROW_DEL_COL) + } +} + + +object MergeExecutor { + + /** + * Spark UI will track all normal accumulators along with Spark tasks to show them on Web UI. + * However, the accumulator used by `MergeIntoCommand` can store a very large value since it + * tracks all files that need to be rewritten. We should ask Spark UI to not remember it, + * otherwise, the UI data may consume lots of memory. Hence, we use the prefix `internal.metrics.` + * to make this accumulator become an internal accumulator, so that it will not be tracked by + * Spark UI. + */ + val TOUCHED_FILES_ACCUM_NAME = "internal.metrics.MergeIntoDelta.touchedFiles" + + val ROW_ID_COL = "_row_id_" + val FILE_PATH_COL: String = GpuDeltaParquetFileFormatUtils.FILE_PATH_COL + val SOURCE_ROW_PRESENT_COL: String = "_source_row_present_" + val SOURCE_ROW_PRESENT_FIELD: StructField = StructField(SOURCE_ROW_PRESENT_COL, BooleanType, + nullable = false) + val TARGET_ROW_PRESENT_COL: String = "_target_row_present_" + val TARGET_ROW_PRESENT_FIELD: StructField = StructField(TARGET_ROW_PRESENT_COL, BooleanType, + nullable = false) + val ROW_DROPPED_COL: String = GpuDeltaMergeConstants.ROW_DROPPED_COL + val ROW_DROPPED_FIELD: StructField = StructField(ROW_DROPPED_COL, BooleanType, nullable = false) + val INCR_METRICS_COL: String = "_incr_metrics_" + val INCR_METRICS_FIELD: StructField = StructField(INCR_METRICS_COL, BooleanType, nullable = false) + val INCR_ROW_COUNT_COL: String = "_incr_row_count_" + + // Some Delta versions use Literal(null) which translates to a literal of NullType instead + // of the Literal(null, StringType) which is needed, so using a fixed version here + // rather than the version from Delta Lake. + val CDC_TYPE_NOT_CDC_LITERAL: Literal = Literal(null, StringType) + + private[delta] def toDeletionVector(bitmap: Roaring64Bitmap): DeletionVectorDescriptor = { + DeletionVectorDescriptor.inlineInLog(RoaringBitmapWrapper(bitmap).serializeToBytes(), + bitmap.getLongCardinality) + } + + /** Count the number of distinct partition values among the AddFiles in the given set. */ + private[delta] def totalBytesAndDistinctPartitionValues(files: Seq[FileAction]): (Long, Int) = { + val distinctValues = new mutable.HashSet[Map[String, String]]() + var bytes = 0L + val iter = files.collect { case a: AddFile => a }.iterator + while (iter.hasNext) { + val file = iter.next() + distinctValues += file.partitionValues + bytes += file.size + } + // If the only distinct value map is an empty map, then it must be an unpartitioned table. + // Return 0 in that case. + val numDistinctValues = + if (distinctValues.size == 1 && distinctValues.head.isEmpty) 0 else distinctValues.size + (bytes, numDistinctValues) + } +} \ No newline at end of file diff --git a/delta-lake/delta-spark341db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuLowShuffleMergeCommand.scala b/delta-lake/delta-spark341db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuLowShuffleMergeCommand.scala new file mode 100644 index 00000000000..fddebda33bd --- /dev/null +++ b/delta-lake/delta-spark341db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuLowShuffleMergeCommand.scala @@ -0,0 +1,1083 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * This file was derived from MergeIntoCommand.scala + * in the Delta Lake project at https://github.com/delta-io/delta. + * + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.databricks.sql.transaction.tahoe.rapids + +import java.net.URI +import java.util.concurrent.TimeUnit + +import scala.collection.mutable + +import com.databricks.sql.io.RowIndexFilterType +import com.databricks.sql.transaction.tahoe._ +import com.databricks.sql.transaction.tahoe.DeltaOperations.MergePredicate +import com.databricks.sql.transaction.tahoe.DeltaParquetFileFormat.DeletionVectorDescriptorWithFilterType +import com.databricks.sql.transaction.tahoe.actions.{AddCDCFile, AddFile, DeletionVectorDescriptor, FileAction} +import com.databricks.sql.transaction.tahoe.commands.DeltaCommand +import com.databricks.sql.transaction.tahoe.rapids.MergeExecutor.{toDeletionVector, totalBytesAndDistinctPartitionValues, FILE_PATH_COL, INCR_METRICS_COL, INCR_METRICS_FIELD, ROW_DROPPED_COL, ROW_DROPPED_FIELD, SOURCE_ROW_PRESENT_COL, SOURCE_ROW_PRESENT_FIELD, TARGET_ROW_PRESENT_COL, TARGET_ROW_PRESENT_FIELD} +import com.databricks.sql.transaction.tahoe.schema.ImplicitMetadataOperation +import com.databricks.sql.transaction.tahoe.sources.DeltaSQLConf +import com.databricks.sql.transaction.tahoe.util.{AnalysisHelper, DeltaFileOperations} +import com.nvidia.spark.rapids.{GpuOverrides, RapidsConf, SparkPlanMeta} +import com.nvidia.spark.rapids.RapidsConf.DELTA_LOW_SHUFFLE_MERGE_DEL_VECTOR_BROADCAST_THRESHOLD +import com.nvidia.spark.rapids.delta._ +import com.nvidia.spark.rapids.delta.GpuDeltaParquetFileFormatUtils.{METADATA_ROW_DEL_COL, METADATA_ROW_DEL_FIELD, METADATA_ROW_IDX_COL, METADATA_ROW_IDX_FIELD} +import com.nvidia.spark.rapids.shims.FileSourceScanExecMeta +import org.roaringbitmap.longlong.Roaring64Bitmap + +import org.apache.spark.SparkContext +import org.apache.spark.internal.Logging +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.expressions.{Alias, And, Attribute, AttributeReference, CaseWhen, Expression, Literal, NamedExpression, PredicateHelper} +import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral +import org.apache.spark.sql.catalyst.plans.logical.{DeltaMergeAction, DeltaMergeIntoClause, DeltaMergeIntoMatchedClause, DeltaMergeIntoMatchedDeleteClause, DeltaMergeIntoMatchedUpdateClause, DeltaMergeIntoNotMatchedBySourceClause, DeltaMergeIntoNotMatchedBySourceDeleteClause, DeltaMergeIntoNotMatchedBySourceUpdateClause, DeltaMergeIntoNotMatchedClause, DeltaMergeIntoNotMatchedInsertClause, LogicalPlan, Project} +import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap +import org.apache.spark.sql.execution.{SparkPlan, SQLExecution} +import org.apache.spark.sql.execution.command.LeafRunnableCommand +import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} +import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.types.{BooleanType, LongType, StringType, StructField, StructType} + +/** + * GPU version of Delta Lake's low shuffle merge implementation. + * + * Performs a merge of a source query/table into a Delta table. + * + * Issues an error message when the ON search_condition of the MERGE statement can match + * a single row from the target table with multiple rows of the source table-reference. + * Different from the original implementation, it optimized writing touched unmodified target files. + * + * Algorithm: + * + * Phase 1: Find the input files in target that are touched by the rows that satisfy + * the condition and verify that no two source rows match with the same target row. + * This is implemented as an inner-join using the given condition. See [[findTouchedFiles]] + * for more details. + * + * Phase 2: Read the touched files again and write new files with updated and/or inserted rows + * without copying unmodified rows. + * + * Phase 3: Read the touched files again and write new files with unmodified rows in target table, + * trying to keep its original order and avoid shuffle as much as possible. + * + * Phase 4: Use the Delta protocol to atomically remove the touched files and add the new files. + * + * @param source Source data to merge from + * @param target Target table to merge into + * @param gpuDeltaLog Delta log to use + * @param condition Condition for a source row to match with a target row + * @param matchedClauses All info related to matched clauses. + * @param notMatchedClauses All info related to not matched clause. + * @param migratedSchema The final schema of the target - may be changed by schema evolution. + */ +case class GpuLowShuffleMergeCommand( + @transient source: LogicalPlan, + @transient target: LogicalPlan, + @transient gpuDeltaLog: GpuDeltaLog, + condition: Expression, + matchedClauses: Seq[DeltaMergeIntoMatchedClause], + notMatchedClauses: Seq[DeltaMergeIntoNotMatchedClause], + notMatchedBySourceClauses: Seq[DeltaMergeIntoNotMatchedBySourceClause], + migratedSchema: Option[StructType])( + @transient val rapidsConf: RapidsConf) + extends LeafRunnableCommand + with DeltaCommand with PredicateHelper with AnalysisHelper with ImplicitMetadataOperation { + + import SQLMetrics._ + + override val otherCopyArgs: Seq[AnyRef] = Seq(rapidsConf) + + override val canMergeSchema: Boolean = conf.getConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE) + override val canOverwriteSchema: Boolean = false + + override val output: Seq[Attribute] = Seq( + AttributeReference("num_affected_rows", LongType)(), + AttributeReference("num_updated_rows", LongType)(), + AttributeReference("num_deleted_rows", LongType)(), + AttributeReference("num_inserted_rows", LongType)()) + + @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() + @transient lazy val targetDeltaLog: DeltaLog = gpuDeltaLog.deltaLog + + override lazy val metrics = Map[String, SQLMetric]( + "numSourceRows" -> createMetric(sc, "number of source rows"), + "numSourceRowsInSecondScan" -> + createMetric(sc, "number of source rows (during repeated scan)"), + "numTargetRowsCopied" -> createMetric(sc, "number of target rows rewritten unmodified"), + "numTargetRowsInserted" -> createMetric(sc, "number of inserted rows"), + "numTargetRowsUpdated" -> createMetric(sc, "number of updated rows"), + "numTargetRowsDeleted" -> createMetric(sc, "number of deleted rows"), + "numTargetRowsMatchedUpdated" -> createMetric(sc, "number of target rows updated when matched"), + "numTargetRowsMatchedDeleted" -> createMetric(sc, "number of target rows deleted when matched"), + "numTargetRowsNotMatchedBySourceUpdated" -> createMetric(sc, + "number of target rows updated when not matched by source"), + "numTargetRowsNotMatchedBySourceDeleted" -> createMetric(sc, + "number of target rows deleted when not matched by source"), + "numTargetFilesBeforeSkipping" -> createMetric(sc, "number of target files before skipping"), + "numTargetFilesAfterSkipping" -> createMetric(sc, "number of target files after skipping"), + "numTargetFilesRemoved" -> createMetric(sc, "number of files removed to target"), + "numTargetFilesAdded" -> createMetric(sc, "number of files added to target"), + "numTargetChangeFilesAdded" -> + createMetric(sc, "number of change data capture files generated"), + "numTargetChangeFileBytes" -> + createMetric(sc, "total size of change data capture files generated"), + "numTargetBytesBeforeSkipping" -> createMetric(sc, "number of target bytes before skipping"), + "numTargetBytesAfterSkipping" -> createMetric(sc, "number of target bytes after skipping"), + "numTargetBytesRemoved" -> createMetric(sc, "number of target bytes removed"), + "numTargetBytesAdded" -> createMetric(sc, "number of target bytes added"), + "numTargetPartitionsAfterSkipping" -> + createMetric(sc, "number of target partitions after skipping"), + "numTargetPartitionsRemovedFrom" -> + createMetric(sc, "number of target partitions from which files were removed"), + "numTargetPartitionsAddedTo" -> + createMetric(sc, "number of target partitions to which files were added"), + "executionTimeMs" -> + createMetric(sc, "time taken to execute the entire operation"), + "scanTimeMs" -> + createMetric(sc, "time taken to scan the files for matches"), + "rewriteTimeMs" -> + createMetric(sc, "time taken to rewrite the matched files")) + + /** Whether this merge statement has only a single insert (NOT MATCHED) clause. */ + protected def isSingleInsertOnly: Boolean = matchedClauses.isEmpty && + notMatchedClauses.length == 1 + + override def run(spark: SparkSession): Seq[Row] = { + recordDeltaOperation(targetDeltaLog, "delta.dml.lowshufflemerge") { + val startTime = System.nanoTime() + val result = gpuDeltaLog.withNewTransaction { deltaTxn => + if (target.schema.size != deltaTxn.metadata.schema.size) { + throw DeltaErrors.schemaChangedSinceAnalysis( + atAnalysis = target.schema, latestSchema = deltaTxn.metadata.schema) + } + + if (canMergeSchema) { + updateMetadata( + spark, deltaTxn, migratedSchema.getOrElse(target.schema), + deltaTxn.metadata.partitionColumns, deltaTxn.metadata.configuration, + isOverwriteMode = false, rearrangeOnly = false) + } + + + val (executor, fallback) = { + val context = MergeExecutorContext(this, spark, deltaTxn, rapidsConf) + if (isSingleInsertOnly && spark.conf.get(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED)) { + (new InsertOnlyMergeExecutor(context), false) + } else { + val executor = new LowShuffleMergeExecutor(context) + (executor, executor.shouldFallback()) + } + } + + if (fallback) { + None + } else { + Some(runLowShuffleMerge(spark, startTime, deltaTxn, executor)) + } + } + + result match { + case Some(row) => row + case None => + // We should rollback to normal gpu + new GpuMergeIntoCommand(source, target, gpuDeltaLog, condition, matchedClauses, + notMatchedClauses, notMatchedBySourceClauses, migratedSchema)(rapidsConf) + .run(spark) + } + } + } + + + private def runLowShuffleMerge( + spark: SparkSession, + startTime: Long, + deltaTxn: GpuOptimisticTransactionBase, + mergeExecutor: MergeExecutor): Seq[Row] = { + val deltaActions = mergeExecutor.execute() + // Metrics should be recorded before commit (where they are written to delta logs). + metrics("executionTimeMs").set((System.nanoTime() - startTime) / 1000 / 1000) + deltaTxn.registerSQLMetrics(spark, metrics) + + // This is a best-effort sanity check. + if (metrics("numSourceRowsInSecondScan").value >= 0 && + metrics("numSourceRows").value != metrics("numSourceRowsInSecondScan").value) { + log.warn(s"Merge source has ${metrics("numSourceRows").value} rows in initial scan but " + + s"${metrics("numSourceRowsInSecondScan").value} rows in second scan") + if (conf.getConf(DeltaSQLConf.MERGE_FAIL_IF_SOURCE_CHANGED)) { + throw DeltaErrors.sourceNotDeterministicInMergeException(spark) + } + } + + deltaTxn.commit( + deltaActions, + DeltaOperations.Merge( + Option(condition), + matchedClauses.map(DeltaOperations.MergePredicate(_)), + notMatchedClauses.map(DeltaOperations.MergePredicate(_)), + // We do not support notMatchedBySourcePredicates yet and fall back to CPU + // See https://github.com/NVIDIA/spark-rapids/issues/8415 + notMatchedBySourcePredicates = Seq.empty[MergePredicate] + )) + + // Record metrics + val stats = GpuMergeStats.fromMergeSQLMetrics( + metrics, + condition, + matchedClauses, + notMatchedClauses, + deltaTxn.metadata.partitionColumns.nonEmpty) + recordDeltaEvent(targetDeltaLog, "delta.dml.merge.stats", data = stats) + + + spark.sharedState.cacheManager.recacheByPlan(spark, target) + + // This is needed to make the SQL metrics visible in the Spark UI. Also this needs + // to be outside the recordMergeOperation because this method will update some metric. + val executionId = spark.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) + SQLMetrics.postDriverMetricUpdates(spark.sparkContext, executionId, metrics.values.toSeq) + Seq(Row(metrics("numTargetRowsUpdated").value + metrics("numTargetRowsDeleted").value + + metrics("numTargetRowsInserted").value, metrics("numTargetRowsUpdated").value, + metrics("numTargetRowsDeleted").value, metrics("numTargetRowsInserted").value)) + } + + /** + * Execute the given `thunk` and return its result while recording the time taken to do it. + * + * @param sqlMetricName name of SQL metric to update with the time taken by the thunk + * @param thunk the code to execute + */ + def recordMergeOperation[A](sqlMetricName: String)(thunk: => A): A = { + val startTimeNs = System.nanoTime() + val r = thunk + val timeTakenMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs) + if (sqlMetricName != null && timeTakenMs > 0) { + metrics(sqlMetricName) += timeTakenMs + } + r + } + + /** Expressions to increment SQL metrics */ + def makeMetricUpdateUDF(name: String, deterministic: Boolean = false) + : Expression = { + // only capture the needed metric in a local variable + val metric = metrics(name) + var u = DeltaUDF.boolean(new GpuDeltaMetricUpdateUDF(metric)) + if (!deterministic) { + u = u.asNondeterministic() + } + u.apply().expr + } +} + +/** + * Context merge execution. + */ +case class MergeExecutorContext(cmd: GpuLowShuffleMergeCommand, + spark: SparkSession, + deltaTxn: OptimisticTransaction, + rapidsConf: RapidsConf) + +trait MergeExecutor extends AnalysisHelper with PredicateHelper with Logging { + + val context: MergeExecutorContext + + + /** + * Map to get target output attributes by name. + * The case sensitivity of the map is set accordingly to Spark configuration. + */ + @transient private lazy val targetOutputAttributesMap: Map[String, Attribute] = { + val attrMap: Map[String, Attribute] = context.cmd.target + .outputSet.view + .map(attr => attr.name -> attr).toMap + if (context.cmd.conf.caseSensitiveAnalysis) { + attrMap + } else { + CaseInsensitiveMap(attrMap) + } + } + + def execute(): Seq[FileAction] + + protected def targetOutputCols: Seq[NamedExpression] = { + context.deltaTxn.metadata.schema.map { col => + targetOutputAttributesMap + .get(col.name) + .map { a => + AttributeReference(col.name, col.dataType, col.nullable)(a.exprId) + } + .getOrElse(Alias(Literal(null), col.name)()) + } + } + + /** + * Build a DataFrame using the given `files` that has the same output columns (exprIds) + * as the `target` logical plan, so that existing update/insert expressions can be applied + * on this new plan. + */ + protected def buildTargetDFWithFiles(files: Seq[AddFile]): DataFrame = { + val targetOutputColsMap = { + val colsMap: Map[String, NamedExpression] = targetOutputCols.view + .map(col => col.name -> col).toMap + if (context.cmd.conf.caseSensitiveAnalysis) { + colsMap + } else { + CaseInsensitiveMap(colsMap) + } + } + + val plan = { + // We have to do surgery to use the attributes from `targetOutputCols` to scan the table. + // In cases of schema evolution, they may not be the same type as the original attributes. + val original = + context.deltaTxn.deltaLog.createDataFrame(context.deltaTxn.snapshot, files) + .queryExecution + .analyzed + val transformed = original.transform { + case LogicalRelation(base, _, catalogTbl, isStreaming) => + LogicalRelation( + base, + // We can ignore the new columns which aren't yet AttributeReferences. + targetOutputCols.collect { case a: AttributeReference => a }, + catalogTbl, + isStreaming) + } + + // In case of schema evolution & column mapping, we would also need to rebuild the file + // format because under column mapping, the reference schema within DeltaParquetFileFormat + // that is used to populate metadata needs to be updated + if (context.deltaTxn.metadata.columnMappingMode != NoMapping) { + val updatedFileFormat = context.deltaTxn.deltaLog.fileFormat( + context.deltaTxn.deltaLog.unsafeVolatileSnapshot.protocol, context.deltaTxn.metadata) + DeltaTableUtils.replaceFileFormat(transformed, updatedFileFormat) + } else { + transformed + } + } + + // For each plan output column, find the corresponding target output column (by name) and + // create an alias + val aliases = plan.output.map { + case newAttrib: AttributeReference => + val existingTargetAttrib = targetOutputColsMap.getOrElse(newAttrib.name, + throw new AnalysisException( + s"Could not find ${newAttrib.name} among the existing target output " + + targetOutputCols.mkString(","))).asInstanceOf[AttributeReference] + + if (existingTargetAttrib.exprId == newAttrib.exprId) { + // It's not valid to alias an expression to its own exprId (this is considered a + // non-unique exprId by the analyzer), so we just use the attribute directly. + newAttrib + } else { + Alias(newAttrib, existingTargetAttrib.name)(exprId = existingTargetAttrib.exprId) + } + } + + Dataset.ofRows(context.spark, Project(aliases, plan)) + } + + + /** + * Repartitions the output DataFrame by the partition columns if table is partitioned + * and `merge.repartitionBeforeWrite.enabled` is set to true. + */ + protected def repartitionIfNeeded(df: DataFrame): DataFrame = { + val partitionColumns = context.deltaTxn.metadata.partitionColumns + // TODO: We should remove this method and use optimized write instead, see + // https://github.com/NVIDIA/spark-rapids/issues/10417 + if (partitionColumns.nonEmpty && context.spark.conf.get(DeltaSQLConf + .MERGE_REPARTITION_BEFORE_WRITE)) { + df.repartition(partitionColumns.map(col): _*) + } else { + df + } + } + + protected def sourceDF: DataFrame = { + // UDF to increment metrics + val incrSourceRowCountExpr = context.cmd.makeMetricUpdateUDF("numSourceRows") + Dataset.ofRows(context.spark, context.cmd.source) + .filter(new Column(incrSourceRowCountExpr)) + } + + /** Whether this merge statement has no insert (NOT MATCHED) clause. */ + protected def hasNoInserts: Boolean = context.cmd.notMatchedClauses.isEmpty + + +} + +/** + * This is an optimization of the case when there is no update clause for the merge. + * We perform an left anti join on the source data to find the rows to be inserted. + * + * This will currently only optimize for the case when there is a _single_ notMatchedClause. + */ +class InsertOnlyMergeExecutor(override val context: MergeExecutorContext) extends MergeExecutor { + override def execute(): Seq[FileAction] = { + context.cmd.recordMergeOperation(sqlMetricName = "rewriteTimeMs") { + + // UDFs to update metrics + val incrSourceRowCountExpr = context.cmd.makeMetricUpdateUDF("numSourceRows") + val incrInsertedCountExpr = context.cmd.makeMetricUpdateUDF("numTargetRowsInserted") + + val outputColNames = targetOutputCols.map(_.name) + // we use head here since we know there is only a single notMatchedClause + val outputExprs = context.cmd.notMatchedClauses.head.resolvedActions.map(_.expr) + val outputCols = outputExprs.zip(outputColNames).map { case (expr, name) => + new Column(Alias(expr, name)()) + } + + // source DataFrame + val sourceDF = Dataset.ofRows(context.spark, context.cmd.source) + .filter(new Column(incrSourceRowCountExpr)) + .filter(new Column(context.cmd.notMatchedClauses.head.condition + .getOrElse(Literal.TrueLiteral))) + + // Skip data based on the merge condition + val conjunctivePredicates = splitConjunctivePredicates(context.cmd.condition) + val targetOnlyPredicates = + conjunctivePredicates.filter(_.references.subsetOf(context.cmd.target.outputSet)) + val dataSkippedFiles = context.deltaTxn.filterFiles(targetOnlyPredicates) + + // target DataFrame + val targetDF = buildTargetDFWithFiles(dataSkippedFiles) + + val insertDf = sourceDF.join(targetDF, new Column(context.cmd.condition), "leftanti") + .select(outputCols: _*) + .filter(new Column(incrInsertedCountExpr)) + + val newFiles = context.deltaTxn + .writeFiles(repartitionIfNeeded(insertDf, + )) + + // Update metrics + context.cmd.metrics("numTargetFilesBeforeSkipping") += context.deltaTxn.snapshot.numOfFiles + context.cmd.metrics("numTargetBytesBeforeSkipping") += context.deltaTxn.snapshot.sizeInBytes + val (afterSkippingBytes, afterSkippingPartitions) = + totalBytesAndDistinctPartitionValues(dataSkippedFiles) + context.cmd.metrics("numTargetFilesAfterSkipping") += dataSkippedFiles.size + context.cmd.metrics("numTargetBytesAfterSkipping") += afterSkippingBytes + context.cmd.metrics("numTargetPartitionsAfterSkipping") += afterSkippingPartitions + context.cmd.metrics("numTargetFilesRemoved") += 0 + context.cmd.metrics("numTargetBytesRemoved") += 0 + context.cmd.metrics("numTargetPartitionsRemovedFrom") += 0 + val (addedBytes, addedPartitions) = totalBytesAndDistinctPartitionValues(newFiles) + context.cmd.metrics("numTargetFilesAdded") += newFiles.count(_.isInstanceOf[AddFile]) + context.cmd.metrics("numTargetBytesAdded") += addedBytes + context.cmd.metrics("numTargetPartitionsAddedTo") += addedPartitions + newFiles + } + } +} + + +/** + * This is an optimized algorithm for merge statement, where we avoid shuffling the unmodified + * target data. + * + * The algorithm is as follows: + * 1. Find touched target files in the target table by joining the source and target data, with + * collecting joined row identifiers as (`__metadata_file_path`, `__metadata_row_idx`) pairs. + * 2. Read the touched files again and write new files with updated and/or inserted rows + * without coping unmodified data from target table, but filtering target table with collected + * rows mentioned above. + * 3. Read the touched files again, filtering unmodified rows with collected row identifiers + * collected in first step, and saving them without shuffle. + */ +class LowShuffleMergeExecutor(override val context: MergeExecutorContext) extends MergeExecutor { + + // We over-count numTargetRowsDeleted when there are multiple matches; + // this is the amount of the overcount, so we can subtract it to get a correct final metric. + private var multipleMatchDeleteOnlyOvercount: Option[Long] = None + + // UDFs to update metrics + private val incrSourceRowCountExpr: Expression = context.cmd. + makeMetricUpdateUDF("numSourceRowsInSecondScan") + private val incrUpdatedCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsUpdated") + private val incrUpdatedMatchedCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsMatchedUpdated") + private val incrUpdatedNotMatchedBySourceCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsNotMatchedBySourceUpdated") + private val incrInsertedCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsInserted") + private val incrDeletedCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsDeleted") + private val incrDeletedMatchedCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsMatchedDeleted") + private val incrDeletedNotMatchedBySourceCountExpr: Expression = context.cmd + .makeMetricUpdateUDF("numTargetRowsNotMatchedBySourceDeleted") + + private def updateOutput(resolvedActions: Seq[DeltaMergeAction], incrExpr: Expression) + : Seq[Expression] = { + resolvedActions.map(_.expr) :+ + Literal.FalseLiteral :+ + UnresolvedAttribute(TARGET_ROW_PRESENT_COL) :+ + UnresolvedAttribute(SOURCE_ROW_PRESENT_COL) :+ + incrExpr + } + + private def deleteOutput(incrExpr: Expression): Seq[Expression] = { + targetOutputCols :+ + TrueLiteral :+ + UnresolvedAttribute(TARGET_ROW_PRESENT_COL) :+ + UnresolvedAttribute(SOURCE_ROW_PRESENT_COL) :+ + incrExpr + } + + private def insertOutput(resolvedActions: Seq[DeltaMergeAction], incrExpr: Expression) + : Seq[Expression] = { + resolvedActions.map(_.expr) :+ + Literal.FalseLiteral :+ + UnresolvedAttribute(TARGET_ROW_PRESENT_COL) :+ + UnresolvedAttribute(SOURCE_ROW_PRESENT_COL) :+ + incrExpr + } + + private def clauseOutput(clause: DeltaMergeIntoClause): Seq[Expression] = clause match { + case u: DeltaMergeIntoMatchedUpdateClause => + updateOutput(u.resolvedActions, And(incrUpdatedCountExpr, incrUpdatedMatchedCountExpr)) + case _: DeltaMergeIntoMatchedDeleteClause => + deleteOutput(And(incrDeletedCountExpr, incrDeletedMatchedCountExpr)) + case i: DeltaMergeIntoNotMatchedInsertClause => + insertOutput(i.resolvedActions, incrInsertedCountExpr) + case u: DeltaMergeIntoNotMatchedBySourceUpdateClause => + updateOutput(u.resolvedActions, + And(incrUpdatedCountExpr, incrUpdatedNotMatchedBySourceCountExpr)) + case _: DeltaMergeIntoNotMatchedBySourceDeleteClause => + deleteOutput(And(incrDeletedCountExpr, incrDeletedNotMatchedBySourceCountExpr)) + } + + private def clauseCondition(clause: DeltaMergeIntoClause): Expression = { + // if condition is None, then expression always evaluates to true + clause.condition.getOrElse(TrueLiteral) + } + + /** + * Though low shuffle merge algorithm performs better than traditional merge algorithm in some + * cases, there are some case we should fallback to traditional merge executor: + * + * 1. Low shuffle merge algorithm requires generating metadata columns such as + * [[METADATA_ROW_IDX_COL]], [[METADATA_ROW_DEL_COL]], which only implemented on + * [[org.apache.spark.sql.rapids.GpuFileSourceScanExec]]. That means we need to fallback to + * this normal executor when [[org.apache.spark.sql.rapids.GpuFileSourceScanExec]] is disabled + * for some reason. + * 2. Low shuffle merge algorithm currently needs to broadcast deletion vector, which may + * introduce extra overhead. It maybe better to fallback to this algorithm when the changeset + * it too large. + */ + def shouldFallback(): Boolean = { + // Trying to detect if we can execute finding touched files. + val touchFilePlanOverrideSucceed = verifyGpuPlan(planForFindingTouchedFiles()) { planMeta => + def check(meta: SparkPlanMeta[SparkPlan]): Boolean = { + meta match { + case scan if scan.isInstanceOf[FileSourceScanExecMeta] => scan + .asInstanceOf[FileSourceScanExecMeta] + .wrapped + .schema + .fieldNames + .contains(METADATA_ROW_IDX_COL) && scan.canThisBeReplaced + case m => m.childPlans.exists(check) + } + } + + check(planMeta) + } + if (!touchFilePlanOverrideSucceed) { + logWarning("Unable to override file scan for low shuffle merge for finding touched files " + + "plan, fallback to tradition merge.") + return true + } + + // Trying to detect if we can execute the merge plan. + val mergePlanOverrideSucceed = verifyGpuPlan(planForMergeExecution(touchedFiles)) { planMeta => + var overrideCount = 0 + def count(meta: SparkPlanMeta[SparkPlan]): Unit = { + meta match { + case scan if scan.isInstanceOf[FileSourceScanExecMeta] => + if (scan.asInstanceOf[FileSourceScanExecMeta] + .wrapped.schema.fieldNames.contains(METADATA_ROW_DEL_COL) && scan.canThisBeReplaced) { + overrideCount += 1 + } + case m => m.childPlans.foreach(count) + } + } + + count(planMeta) + overrideCount == 2 + } + + if (!mergePlanOverrideSucceed) { + logWarning("Unable to override file scan for low shuffle merge for merge plan, fallback to " + + "tradition merge.") + return true + } + + val deletionVectorSize = touchedFiles.values.map(_._1.serializedSizeInBytes()).sum + val maxDelVectorSize = context.rapidsConf + .get(DELTA_LOW_SHUFFLE_MERGE_DEL_VECTOR_BROADCAST_THRESHOLD) + if (deletionVectorSize > maxDelVectorSize) { + logWarning( + s"""Low shuffle merge can't be executed because broadcast deletion vector count + |$deletionVectorSize is large than max value $maxDelVectorSize """.stripMargin) + return true + } + + false + } + + private def verifyGpuPlan(input: DataFrame)(checkPlanMeta: SparkPlanMeta[SparkPlan] => Boolean) + : Boolean = { + val overridePlan = GpuOverrides.wrapAndTagPlan(input.queryExecution.sparkPlan, + context.rapidsConf) + checkPlanMeta(overridePlan) + } + + override def execute(): Seq[FileAction] = { + val newFiles = context.cmd.withStatusCode("DELTA", + s"Rewriting ${touchedFiles.size} files and saving modified data") { + val df = planForMergeExecution(touchedFiles) + context.deltaTxn.writeFiles(df) + } + + // Update metrics + val (addedBytes, addedPartitions) = totalBytesAndDistinctPartitionValues(newFiles) + context.cmd.metrics("numTargetFilesAdded") += newFiles.count(_.isInstanceOf[AddFile]) + context.cmd.metrics("numTargetChangeFilesAdded") += newFiles.count(_.isInstanceOf[AddCDCFile]) + context.cmd.metrics("numTargetChangeFileBytes") += newFiles.collect { + case f: AddCDCFile => f.size + } + .sum + context.cmd.metrics("numTargetBytesAdded") += addedBytes + context.cmd.metrics("numTargetPartitionsAddedTo") += addedPartitions + + if (multipleMatchDeleteOnlyOvercount.isDefined) { + // Compensate for counting duplicates during the query. + val actualRowsDeleted = + context.cmd.metrics("numTargetRowsDeleted").value - multipleMatchDeleteOnlyOvercount.get + assert(actualRowsDeleted >= 0) + context.cmd.metrics("numTargetRowsDeleted").set(actualRowsDeleted) + } + + touchedFiles.values.map(_._2).map(_.remove).toSeq ++ newFiles + } + + private lazy val dataSkippedFiles: Seq[AddFile] = { + // Skip data based on the merge condition + val targetOnlyPredicates = splitConjunctivePredicates(context.cmd.condition) + .filter(_.references.subsetOf(context.cmd.target.outputSet)) + context.deltaTxn.filterFiles(targetOnlyPredicates) + } + + private lazy val dataSkippedTargetDF: DataFrame = { + addRowIndexMetaColumn(buildTargetDFWithFiles(dataSkippedFiles)) + } + + private lazy val touchedFiles: Map[String, (Roaring64Bitmap, AddFile)] = this.findTouchedFiles() + + private def planForFindingTouchedFiles(): DataFrame = { + + // Apply inner join to between source and target using the merge condition to find matches + // In addition, we attach two columns + // - METADATA_ROW_IDX column to identify target row in file + // - FILE_PATH_COL the target file name the row is from to later identify the files touched + // by matched rows + val targetDF = dataSkippedTargetDF.withColumn(FILE_PATH_COL, input_file_name()) + + sourceDF.join(targetDF, new Column(context.cmd.condition), "inner") + } + + private def planForMergeExecution(touchedFiles: Map[String, (Roaring64Bitmap, AddFile)]) + : DataFrame = { + getModifiedDF(touchedFiles).unionAll(getUnmodifiedDF(touchedFiles)) + } + + /** + * Find the target table files that contain the rows that satisfy the merge condition. This is + * implemented as an inner-join between the source query/table and the target table using + * the merge condition. + */ + private def findTouchedFiles(): Map[String, (Roaring64Bitmap, AddFile)] = + context.cmd.recordMergeOperation(sqlMetricName = "scanTimeMs") { + context.spark.udf.register("row_index_set", udaf(RoaringBitmapUDAF)) + // Process the matches from the inner join to record touched files and find multiple matches + val collectTouchedFiles = planForFindingTouchedFiles() + .select(col(FILE_PATH_COL), col(METADATA_ROW_IDX_COL)) + .groupBy(FILE_PATH_COL) + .agg( + expr(s"row_index_set($METADATA_ROW_IDX_COL) as row_idxes"), + count("*").as("count")) + .collect().map(row => { + val filename = row.getAs[String](FILE_PATH_COL) + val rowIdxSet = row.getAs[RoaringBitmapWrapper]("row_idxes").inner + val count = row.getAs[Long]("count") + (filename, (rowIdxSet, count)) + }) + .toMap + + val duplicateCount = { + val distinctMatchedRowCounts = collectTouchedFiles.values + .map(_._1.getLongCardinality).sum + val allMatchedRowCounts = collectTouchedFiles.values.map(_._2).sum + allMatchedRowCounts - distinctMatchedRowCounts + } + + val hasMultipleMatches = duplicateCount > 0 + + // Throw error if multiple matches are ambiguous or cannot be computed correctly. + val canBeComputedUnambiguously = { + // Multiple matches are not ambiguous when there is only one unconditional delete as + // all the matched row pairs in the 2nd join in `writeAllChanges` will get deleted. + val isUnconditionalDelete = context.cmd.matchedClauses.headOption match { + case Some(DeltaMergeIntoMatchedDeleteClause(None)) => true + case _ => false + } + context.cmd.matchedClauses.size == 1 && isUnconditionalDelete + } + + if (hasMultipleMatches && !canBeComputedUnambiguously) { + throw DeltaErrors.multipleSourceRowMatchingTargetRowInMergeException(context.spark) + } + + if (hasMultipleMatches) { + // This is only allowed for delete-only queries. + // This query will count the duplicates for numTargetRowsDeleted in Job 2, + // because we count matches after the join and not just the target rows. + // We have to compensate for this by subtracting the duplicates later, + // so we need to record them here. + multipleMatchDeleteOnlyOvercount = Some(duplicateCount) + } + + // Get the AddFiles using the touched file names. + val touchedFileNames = collectTouchedFiles.keys.toSeq + + val nameToAddFileMap = context.cmd.generateCandidateFileMap( + context.cmd.targetDeltaLog.dataPath, + dataSkippedFiles) + + val touchedAddFiles = touchedFileNames.map(f => + context.cmd.getTouchedFile(context.cmd.targetDeltaLog.dataPath, f, nameToAddFileMap)) + .map(f => (DeltaFileOperations + .absolutePath(context.cmd.targetDeltaLog.dataPath.toString, f.path) + .toString, f)).toMap + + // When the target table is empty, and the optimizer optimized away the join entirely + // numSourceRows will be incorrectly 0. + // We need to scan the source table once to get the correct + // metric here. + if (context.cmd.metrics("numSourceRows").value == 0 && + (dataSkippedFiles.isEmpty || dataSkippedTargetDF.take(1).isEmpty)) { + val numSourceRows = sourceDF.count() + context.cmd.metrics("numSourceRows").set(numSourceRows) + } + + // Update metrics + context.cmd.metrics("numTargetFilesBeforeSkipping") += context.deltaTxn.snapshot.numOfFiles + context.cmd.metrics("numTargetBytesBeforeSkipping") += context.deltaTxn.snapshot.sizeInBytes + val (afterSkippingBytes, afterSkippingPartitions) = + totalBytesAndDistinctPartitionValues(dataSkippedFiles) + context.cmd.metrics("numTargetFilesAfterSkipping") += dataSkippedFiles.size + context.cmd.metrics("numTargetBytesAfterSkipping") += afterSkippingBytes + context.cmd.metrics("numTargetPartitionsAfterSkipping") += afterSkippingPartitions + val (removedBytes, removedPartitions) = + totalBytesAndDistinctPartitionValues(touchedAddFiles.values.toSeq) + context.cmd.metrics("numTargetFilesRemoved") += touchedAddFiles.size + context.cmd.metrics("numTargetBytesRemoved") += removedBytes + context.cmd.metrics("numTargetPartitionsRemovedFrom") += removedPartitions + + collectTouchedFiles.map(kv => (kv._1, (kv._2._1, touchedAddFiles(kv._1)))) + } + + + /** + * Modify original data frame to insert + * [[GpuDeltaParquetFileFormatUtils.METADATA_ROW_IDX_COL]]. + */ + private def addRowIndexMetaColumn(baseDF: DataFrame): DataFrame = { + val rowIdxAttr = AttributeReference( + METADATA_ROW_IDX_COL, + METADATA_ROW_IDX_FIELD.dataType, + METADATA_ROW_IDX_FIELD.nullable)() + + val newPlan = baseDF.queryExecution.analyzed.transformUp { + case r@LogicalRelation(fs: HadoopFsRelation, _, _, _) => + val newSchema = StructType(fs.dataSchema.fields).add(METADATA_ROW_IDX_FIELD) + + // This is required to ensure that row index is correctly calculated. + val newFileFormat = fs.fileFormat.asInstanceOf[DeltaParquetFileFormat] + .copy(isSplittable = false, disablePushDowns = true) + + val newFs = fs.copy(dataSchema = newSchema, fileFormat = newFileFormat)(context.spark) + + val newOutput = r.output :+ rowIdxAttr + r.copy(relation = newFs, output = newOutput) + case p@Project(projectList, _) => + val newProjectList = projectList :+ rowIdxAttr + p.copy(projectList = newProjectList) + } + + Dataset.ofRows(context.spark, newPlan) + } + + /** + * The result is scanning target table with touched files, and added an extra + * [[METADATA_ROW_DEL_COL]] to indicate whether filtered by joining with source table in first + * step. + */ + private def getTouchedTargetDF(touchedFiles: Map[String, (Roaring64Bitmap, AddFile)]) + : DataFrame = { + // Generate a new target dataframe that has same output attributes exprIds as the target plan. + // This allows us to apply the existing resolved update/insert expressions. + val baseTargetDF = buildTargetDFWithFiles(touchedFiles.values.map(_._2).toSeq) + + val newPlan = { + val rowDelAttr = AttributeReference( + METADATA_ROW_DEL_COL, + METADATA_ROW_DEL_FIELD.dataType, + METADATA_ROW_DEL_FIELD.nullable)() + + baseTargetDF.queryExecution.analyzed.transformUp { + case r@LogicalRelation(fs: HadoopFsRelation, _, _, _) => + val newSchema = StructType(fs.dataSchema.fields).add(METADATA_ROW_DEL_FIELD) + + // This is required to ensure that row index is correctly calculated. + val newFileFormat = { + val oldFormat = fs.fileFormat.asInstanceOf[DeltaParquetFileFormat] + val dvs = touchedFiles.map(kv => (new URI(kv._1), + DeletionVectorDescriptorWithFilterType(toDeletionVector(kv._2._1), + RowIndexFilterType.UNKNOWN))) + val broadcastDVs = context.spark.sparkContext.broadcast(dvs) + + oldFormat.copy(isSplittable = false, + broadcastDvMap = Some(broadcastDVs), + disablePushDowns = true) + } + + val newFs = fs.copy(dataSchema = newSchema, fileFormat = newFileFormat)(context.spark) + + val newOutput = r.output :+ rowDelAttr + r.copy(relation = newFs, output = newOutput) + case p@Project(projectList, _) => + val newProjectList = projectList :+ rowDelAttr + p.copy(projectList = newProjectList) + } + } + + val df = Dataset.ofRows(context.spark, newPlan) + .withColumn(TARGET_ROW_PRESENT_COL, lit(true)) + + df + } + + /** + * Generate a plan by calculating modified rows. It's computed by joining source and target + * tables, where target table has been filtered by (`__metadata_file_name`, + * `__metadata_row_idx`) pairs collected in first step. + * + * Schema of `modifiedDF`: + * + * targetSchema + ROW_DROPPED_COL + TARGET_ROW_PRESENT_COL + + * SOURCE_ROW_PRESENT_COL + INCR_METRICS_COL + * INCR_METRICS_COL + * + * It consists of several parts: + * + * 1. Unmatched source rows which are inserted + * 2. Unmatched source rows which are deleted + * 3. Target rows which are updated + * 4. Target rows which are deleted + */ + private def getModifiedDF(touchedFiles: Map[String, (Roaring64Bitmap, AddFile)]): DataFrame = { + val sourceDF = this.sourceDF + .withColumn(SOURCE_ROW_PRESENT_COL, new Column(incrSourceRowCountExpr)) + + val targetDF = getTouchedTargetDF(touchedFiles) + + val joinedDF = { + val joinType = if (hasNoInserts && + context.spark.conf.get(DeltaSQLConf.MERGE_MATCHED_ONLY_ENABLED)) { + "inner" + } else { + "leftOuter" + } + val matchedTargetDF = targetDF.filter(METADATA_ROW_DEL_COL) + .drop(METADATA_ROW_DEL_COL) + + sourceDF.join(matchedTargetDF, new Column(context.cmd.condition), joinType) + } + + val modifiedRowsSchema = context.deltaTxn.metadata.schema + .add(ROW_DROPPED_FIELD) + .add(TARGET_ROW_PRESENT_FIELD.copy(nullable = true)) + .add(SOURCE_ROW_PRESENT_FIELD.copy(nullable = true)) + .add(INCR_METRICS_FIELD) + + // Here we generate a case when statement to handle all cases: + // CASE + // WHEN + // CASE WHEN + // + // WHEN + // + // ELSE + // + // WHEN + // CASE WHEN + // + // WHEN + // + // ELSE + // + // END + + val notMatchedConditions = context.cmd.notMatchedClauses.map(clauseCondition) + val notMatchedExpr = { + val deletedNotMatchedRow = { + targetOutputCols :+ + Literal.TrueLiteral :+ + Literal.FalseLiteral :+ + Literal(null) :+ + Literal.TrueLiteral + } + if (context.cmd.notMatchedClauses.isEmpty) { + // If there no `WHEN NOT MATCHED` clause, we should just delete not matched row + deletedNotMatchedRow + } else { + val notMatchedOutputs = context.cmd.notMatchedClauses.map(clauseOutput) + modifiedRowsSchema.zipWithIndex.map { + case (_, idx) => + CaseWhen(notMatchedConditions.zip(notMatchedOutputs.map(_(idx))), + deletedNotMatchedRow(idx)) + } + } + } + + val matchedConditions = context.cmd.matchedClauses.map(clauseCondition) + val matchedOutputs = context.cmd.matchedClauses.map(clauseOutput) + val matchedExprs = { + val notMatchedRow = { + targetOutputCols :+ + Literal.FalseLiteral :+ + Literal.TrueLiteral :+ + Literal(null) :+ + Literal.TrueLiteral + } + if (context.cmd.matchedClauses.isEmpty) { + // If there is not matched clause, this is insert only, we should delete this row. + notMatchedRow + } else { + modifiedRowsSchema.zipWithIndex.map { + case (_, idx) => + CaseWhen(matchedConditions.zip(matchedOutputs.map(_(idx))), + notMatchedRow(idx)) + } + } + } + + val sourceRowHasNoMatch = col(TARGET_ROW_PRESENT_COL).isNull.expr + + val modifiedCols = modifiedRowsSchema.zipWithIndex.map { case (col, idx) => + val caseWhen = CaseWhen( + Seq(sourceRowHasNoMatch -> notMatchedExpr(idx)), + matchedExprs(idx)) + new Column(Alias(caseWhen, col.name)()) + } + + val modifiedDF = { + + // Make this a udf to avoid catalyst to be too aggressive to even remove the join! + val noopRowDroppedCol = udf(new GpuDeltaNoopUDF()).apply(!col(ROW_DROPPED_COL)) + + val modifiedDF = joinedDF.select(modifiedCols: _*) + // This will not filter anything since they always return true, but we need to avoid + // catalyst from optimizing these udf + .filter(noopRowDroppedCol && col(INCR_METRICS_COL)) + .drop(ROW_DROPPED_COL, INCR_METRICS_COL, TARGET_ROW_PRESENT_COL, SOURCE_ROW_PRESENT_COL) + + repartitionIfNeeded(modifiedDF) + } + + modifiedDF + } + + private def getUnmodifiedDF(touchedFiles: Map[String, (Roaring64Bitmap, AddFile)]): DataFrame = { + getTouchedTargetDF(touchedFiles) + .filter(!col(METADATA_ROW_DEL_COL)) + .drop(TARGET_ROW_PRESENT_COL, METADATA_ROW_DEL_COL) + } +} + + +object MergeExecutor { + + /** + * Spark UI will track all normal accumulators along with Spark tasks to show them on Web UI. + * However, the accumulator used by `MergeIntoCommand` can store a very large value since it + * tracks all files that need to be rewritten. We should ask Spark UI to not remember it, + * otherwise, the UI data may consume lots of memory. Hence, we use the prefix `internal.metrics.` + * to make this accumulator become an internal accumulator, so that it will not be tracked by + * Spark UI. + */ + val TOUCHED_FILES_ACCUM_NAME = "internal.metrics.MergeIntoDelta.touchedFiles" + + val ROW_ID_COL = "_row_id_" + val FILE_PATH_COL: String = GpuDeltaParquetFileFormatUtils.FILE_PATH_COL + val SOURCE_ROW_PRESENT_COL: String = "_source_row_present_" + val SOURCE_ROW_PRESENT_FIELD: StructField = StructField(SOURCE_ROW_PRESENT_COL, BooleanType, + nullable = false) + val TARGET_ROW_PRESENT_COL: String = "_target_row_present_" + val TARGET_ROW_PRESENT_FIELD: StructField = StructField(TARGET_ROW_PRESENT_COL, BooleanType, + nullable = false) + val ROW_DROPPED_COL: String = GpuDeltaMergeConstants.ROW_DROPPED_COL + val ROW_DROPPED_FIELD: StructField = StructField(ROW_DROPPED_COL, BooleanType, nullable = false) + val INCR_METRICS_COL: String = "_incr_metrics_" + val INCR_METRICS_FIELD: StructField = StructField(INCR_METRICS_COL, BooleanType, nullable = false) + val INCR_ROW_COUNT_COL: String = "_incr_row_count_" + + // Some Delta versions use Literal(null) which translates to a literal of NullType instead + // of the Literal(null, StringType) which is needed, so using a fixed version here + // rather than the version from Delta Lake. + val CDC_TYPE_NOT_CDC_LITERAL: Literal = Literal(null, StringType) + + def toDeletionVector(bitmap: Roaring64Bitmap): DeletionVectorDescriptor = { + DeletionVectorDescriptor.inlineInLog(RoaringBitmapWrapper(bitmap).serializeToBytes(), + bitmap.getLongCardinality) + } + + /** Count the number of distinct partition values among the AddFiles in the given set. */ + def totalBytesAndDistinctPartitionValues(files: Seq[FileAction]): (Long, Int) = { + val distinctValues = new mutable.HashSet[Map[String, String]]() + var bytes = 0L + val iter = files.collect { case a: AddFile => a }.iterator + while (iter.hasNext) { + val file = iter.next() + distinctValues += file.partitionValues + bytes += file.size + } + // If the only distinct value map is an empty map, then it must be an unpartitioned table. + // Return 0 in that case. + val numDistinctValues = + if (distinctValues.size == 1 && distinctValues.head.isEmpty) 0 else distinctValues.size + (bytes, numDistinctValues) + } +} \ No newline at end of file diff --git a/delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormat.scala b/delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormat.scala index 969d005b573..604ed826397 100644 --- a/delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormat.scala +++ b/delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormat.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,20 +16,32 @@ package com.nvidia.spark.rapids.delta +import java.net.URI + import com.databricks.sql.transaction.tahoe.{DeltaColumnMappingMode, DeltaParquetFileFormat, IdMapping} -import com.databricks.sql.transaction.tahoe.DeltaParquetFileFormat.IS_ROW_DELETED_COLUMN_NAME -import com.nvidia.spark.rapids.SparkPlanMeta +import com.databricks.sql.transaction.tahoe.DeltaParquetFileFormat.{DeletionVectorDescriptorWithFilterType, IS_ROW_DELETED_COLUMN_NAME} +import com.nvidia.spark.rapids.{GpuMetric, RapidsConf, SparkPlanMeta} +import com.nvidia.spark.rapids.delta.GpuDeltaParquetFileFormatUtils.addMetadataColumnToIterator +import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path +import org.apache.spark.broadcast.Broadcast import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.execution.FileSourceScanExec +import org.apache.spark.sql.execution.datasources.PartitionedFile import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.sources.Filter import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.vectorized.ColumnarBatch case class GpuDeltaParquetFileFormat( override val columnMappingMode: DeltaColumnMappingMode, override val referenceSchema: StructType, - isSplittable: Boolean) extends GpuDeltaParquetFileFormatBase { + isSplittable: Boolean, + disablePushDown: Boolean, + broadcastDvMap: Option[Broadcast[Map[URI, DeletionVectorDescriptorWithFilterType]]] +) extends GpuDeltaParquetFileFormatBase { if (columnMappingMode == IdMapping) { val requiredReadConf = SQLConf.PARQUET_FIELD_ID_READ_ENABLED @@ -44,6 +56,46 @@ case class GpuDeltaParquetFileFormat( sparkSession: SparkSession, options: Map[String, String], path: Path): Boolean = isSplittable + + override def buildReaderWithPartitionValuesAndMetrics( + sparkSession: SparkSession, + dataSchema: StructType, + partitionSchema: StructType, + requiredSchema: StructType, + filters: Seq[Filter], + options: Map[String, String], + hadoopConf: Configuration, + metrics: Map[String, GpuMetric], + alluxioPathReplacementMap: Option[Map[String, String]]) + : PartitionedFile => Iterator[InternalRow] = { + + val dataReader = super.buildReaderWithPartitionValuesAndMetrics( + sparkSession, + dataSchema, + partitionSchema, + requiredSchema, + filters, + options, + hadoopConf, + metrics, + alluxioPathReplacementMap) + + val delVecs = broadcastDvMap + val maxDelVecScatterBatchSize = RapidsConf + .DELTA_LOW_SHUFFLE_MERGE_SCATTER_DEL_VECTOR_BATCH_SIZE + .get(sparkSession.sessionState.conf) + + (file: PartitionedFile) => { + val input = dataReader(file) + val dv = delVecs.flatMap(_.value.get(new URI(file.filePath.toString()))) + .map(dv => RoaringBitmapWrapper.deserializeFromBytes(dv.descriptor.inlineData).inner) + addMetadataColumnToIterator(prepareSchema(requiredSchema), + dv, + input.asInstanceOf[Iterator[ColumnarBatch]], + maxDelVecScatterBatchSize + ).asInstanceOf[Iterator[InternalRow]] + } + } } object GpuDeltaParquetFileFormat { @@ -60,6 +112,7 @@ object GpuDeltaParquetFileFormat { } def convertToGpu(fmt: DeltaParquetFileFormat): GpuDeltaParquetFileFormat = { - GpuDeltaParquetFileFormat(fmt.columnMappingMode, fmt.referenceSchema, fmt.isSplittable) + GpuDeltaParquetFileFormat(fmt.columnMappingMode, fmt.referenceSchema, fmt.isSplittable, + fmt.disablePushDowns, fmt.broadcastDvMap) } } diff --git a/delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/shims/MergeIntoCommandMetaShim.scala b/delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/shims/MergeIntoCommandMetaShim.scala index 8e13a9e4b5a..5a2b4e7b52e 100644 --- a/delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/shims/MergeIntoCommandMetaShim.scala +++ b/delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/shims/MergeIntoCommandMetaShim.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,13 +17,14 @@ package com.nvidia.spark.rapids.delta.shims import com.databricks.sql.transaction.tahoe.commands.{MergeIntoCommand, MergeIntoCommandEdge} -import com.databricks.sql.transaction.tahoe.rapids.{GpuDeltaLog, GpuMergeIntoCommand} -import com.nvidia.spark.rapids.RapidsConf +import com.databricks.sql.transaction.tahoe.rapids.{GpuDeltaLog, GpuLowShuffleMergeCommand, GpuMergeIntoCommand} +import com.nvidia.spark.rapids.{RapidsConf, RapidsReaderType} import com.nvidia.spark.rapids.delta.{MergeIntoCommandEdgeMeta, MergeIntoCommandMeta} +import org.apache.spark.internal.Logging import org.apache.spark.sql.execution.command.RunnableCommand -object MergeIntoCommandMetaShim { +object MergeIntoCommandMetaShim extends Logging { def tagForGpu(meta: MergeIntoCommandMeta, mergeCmd: MergeIntoCommand): Unit = { // see https://github.com/NVIDIA/spark-rapids/issues/8415 for more information if (mergeCmd.notMatchedBySourceClauses.nonEmpty) { @@ -39,26 +40,82 @@ object MergeIntoCommandMetaShim { } def convertToGpu(mergeCmd: MergeIntoCommand, conf: RapidsConf): RunnableCommand = { - GpuMergeIntoCommand( - mergeCmd.source, - mergeCmd.target, - new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), - mergeCmd.condition, - mergeCmd.matchedClauses, - mergeCmd.notMatchedClauses, - mergeCmd.notMatchedBySourceClauses, - mergeCmd.migratedSchema)(conf) + // TODO: Currently we only support low shuffler merge only when parquet per file read is enabled + // due to the limitation of implementing row index metadata column. + if (conf.isDeltaLowShuffleMergeEnabled) { + if (conf.isParquetPerFileReadEnabled) { + GpuLowShuffleMergeCommand( + mergeCmd.source, + mergeCmd.target, + new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), + mergeCmd.condition, + mergeCmd.matchedClauses, + mergeCmd.notMatchedClauses, + mergeCmd.notMatchedBySourceClauses, + mergeCmd.migratedSchema)(conf) + } else { + logWarning(s"""Low shuffle merge disabled since ${RapidsConf.PARQUET_READER_TYPE} is + not set to ${RapidsReaderType.PERFILE}. Falling back to classic merge.""") + GpuMergeIntoCommand( + mergeCmd.source, + mergeCmd.target, + new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), + mergeCmd.condition, + mergeCmd.matchedClauses, + mergeCmd.notMatchedClauses, + mergeCmd.notMatchedBySourceClauses, + mergeCmd.migratedSchema)(conf) + } + } else { + GpuMergeIntoCommand( + mergeCmd.source, + mergeCmd.target, + new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), + mergeCmd.condition, + mergeCmd.matchedClauses, + mergeCmd.notMatchedClauses, + mergeCmd.notMatchedBySourceClauses, + mergeCmd.migratedSchema)(conf) + } } def convertToGpu(mergeCmd: MergeIntoCommandEdge, conf: RapidsConf): RunnableCommand = { - GpuMergeIntoCommand( - mergeCmd.source, - mergeCmd.target, - new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), - mergeCmd.condition, - mergeCmd.matchedClauses, - mergeCmd.notMatchedClauses, - mergeCmd.notMatchedBySourceClauses, - mergeCmd.migratedSchema)(conf) + // TODO: Currently we only support low shuffler merge only when parquet per file read is enabled + // due to the limitation of implementing row index metadata column. + if (conf.isDeltaLowShuffleMergeEnabled) { + if (conf.isParquetPerFileReadEnabled) { + GpuLowShuffleMergeCommand( + mergeCmd.source, + mergeCmd.target, + new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), + mergeCmd.condition, + mergeCmd.matchedClauses, + mergeCmd.notMatchedClauses, + mergeCmd.notMatchedBySourceClauses, + mergeCmd.migratedSchema)(conf) + } else { + logWarning(s"""Low shuffle merge is still disable since ${RapidsConf.PARQUET_READER_TYPE} is + not set to ${RapidsReaderType.PERFILE}. Falling back to classic merge.""") + GpuMergeIntoCommand( + mergeCmd.source, + mergeCmd.target, + new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), + mergeCmd.condition, + mergeCmd.matchedClauses, + mergeCmd.notMatchedClauses, + mergeCmd.notMatchedBySourceClauses, + mergeCmd.migratedSchema)(conf) + } + } else { + GpuMergeIntoCommand( + mergeCmd.source, + mergeCmd.target, + new GpuDeltaLog(mergeCmd.targetFileIndex.deltaLog, conf), + mergeCmd.condition, + mergeCmd.matchedClauses, + mergeCmd.notMatchedClauses, + mergeCmd.notMatchedBySourceClauses, + mergeCmd.migratedSchema)(conf) + } } } diff --git a/docs/additional-functionality/advanced_configs.md b/docs/additional-functionality/advanced_configs.md index 3231b7b3069..941ab4046e6 100644 --- a/docs/additional-functionality/advanced_configs.md +++ b/docs/additional-functionality/advanced_configs.md @@ -73,6 +73,12 @@ Name | Description | Default Value | Applicable at spark.rapids.sql.csv.read.double.enabled|CSV reading is not 100% compatible when reading doubles.|true|Runtime spark.rapids.sql.csv.read.float.enabled|CSV reading is not 100% compatible when reading floats.|true|Runtime spark.rapids.sql.decimalOverflowGuarantees|FOR TESTING ONLY. DO NOT USE IN PRODUCTION. Please see the decimal section of the compatibility documents for more information on this config.|true|Runtime +spark.rapids.sql.delta.lowShuffleMerge.deletionVector.broadcast.threshold|Currently we need to broadcast deletion vector to all executors to perform low shuffle merge. When we detect the deletion vector broadcast size is larger than this value, we will fallback to normal shuffle merge.|20971520|Runtime +spark.rapids.sql.delta.lowShuffleMerge.enabled|Option to turn on the low shuffle merge for Delta Lake. Currently there are some limitations for this feature: +1. We only support Databricks Runtime 13.3 and Deltalake 2.4. +2. The file scan mode must be set to PERFILE +3. The deletion vector size must be smaller than spark.rapids.sql.delta.lowShuffleMerge.deletionVector.broadcast.threshold +|false|Runtime spark.rapids.sql.detectDeltaCheckpointQueries|Queries against Delta Lake _delta_log checkpoint Parquet files are not efficient on the GPU. When this option is enabled, the plugin will attempt to detect these queries and fall back to the CPU.|true|Runtime spark.rapids.sql.detectDeltaLogQueries|Queries against Delta Lake _delta_log JSON files are not efficient on the GPU. When this option is enabled, the plugin will attempt to detect these queries and fall back to the CPU.|true|Runtime spark.rapids.sql.fast.sample|Option to turn on fast sample. If enable it is inconsistent with CPU sample because of GPU sample algorithm is inconsistent with CPU.|false|Runtime diff --git a/integration_tests/src/main/python/delta_lake_low_shuffle_merge_test.py b/integration_tests/src/main/python/delta_lake_low_shuffle_merge_test.py new file mode 100644 index 00000000000..6935ee13751 --- /dev/null +++ b/integration_tests/src/main/python/delta_lake_low_shuffle_merge_test.py @@ -0,0 +1,165 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pyspark.sql.functions as f +import pytest + +from delta_lake_merge_common import * +from marks import * +from pyspark.sql.types import * +from spark_session import is_databricks133_or_later, spark_version + +delta_merge_enabled_conf = copy_and_update(delta_writes_enabled_conf, + {"spark.rapids.sql.command.MergeIntoCommand": "true", + "spark.rapids.sql.command.MergeIntoCommandEdge": "true", + "spark.rapids.sql.delta.lowShuffleMerge.enabled": "true", + "spark.rapids.sql.format.parquet.reader.type": "PERFILE"}) + +@allow_non_gpu("ColumnarToRowExec", *delta_meta_allow) +@delta_lake +@ignore_order +@pytest.mark.skipif(not ((is_databricks_runtime() and is_databricks133_or_later()) or + (not is_databricks_runtime() and spark_version().startswith("3.4"))), + reason="Delta Lake Low Shuffle Merge only supports Databricks 13.3 or OSS " + "delta 2.4") +@pytest.mark.parametrize("use_cdf", [True, False], ids=idfn) +@pytest.mark.parametrize("num_slices", num_slices_to_test, ids=idfn) +def test_delta_low_shuffle_merge_when_gpu_file_scan_override_failed(spark_tmp_path, + spark_tmp_table_factory, + use_cdf, num_slices): + # Need to eliminate duplicate keys in the source table otherwise update semantics are ambiguous + src_table_func = lambda spark: two_col_df(spark, int_gen, string_gen, num_slices=num_slices).groupBy("a").agg(f.max("b").alias("b")) + dest_table_func = lambda spark: two_col_df(spark, int_gen, string_gen, seed=1, num_slices=num_slices) + merge_sql = "MERGE INTO {dest_table} USING {src_table} ON {dest_table}.a == {src_table}.a" \ + " WHEN MATCHED THEN UPDATE SET * WHEN NOT MATCHED THEN INSERT *" + + conf = copy_and_update(delta_merge_enabled_conf, + { + "spark.rapids.sql.exec.FileSourceScanExec": "false", + # Disable auto broadcast join due to this issue: + # https://github.com/NVIDIA/spark-rapids/issues/10973 + "spark.sql.autoBroadcastJoinThreshold": "-1" + }) + assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, + src_table_func, dest_table_func, merge_sql, False, conf=conf) + + + +@allow_non_gpu(*delta_meta_allow) +@delta_lake +@ignore_order +@pytest.mark.skipif(not ((is_databricks_runtime() and is_databricks133_or_later()) or + (not is_databricks_runtime() and spark_version().startswith("3.4"))), + reason="Delta Lake Low Shuffle Merge only supports Databricks 13.3 or OSS " + "delta 2.4") +@pytest.mark.parametrize("table_ranges", [(range(20), range(10)), # partial insert of source + (range(5), range(5)), # no-op insert + (range(10), range(20, 30)) # full insert of source + ], ids=idfn) +@pytest.mark.parametrize("use_cdf", [True, False], ids=idfn) +@pytest.mark.parametrize("partition_columns", [None, ["a"], ["b"], ["a", "b"]], ids=idfn) +@pytest.mark.parametrize("num_slices", num_slices_to_test, ids=idfn) +def test_delta_merge_not_match_insert_only(spark_tmp_path, spark_tmp_table_factory, table_ranges, + use_cdf, partition_columns, num_slices): + do_test_delta_merge_not_match_insert_only(spark_tmp_path, spark_tmp_table_factory, + table_ranges, use_cdf, partition_columns, + num_slices, False, delta_merge_enabled_conf) + +@allow_non_gpu(*delta_meta_allow) +@delta_lake +@ignore_order +@pytest.mark.skipif(not ((is_databricks_runtime() and is_databricks133_or_later()) or + (not is_databricks_runtime() and spark_version().startswith("3.4"))), + reason="Delta Lake Low Shuffle Merge only supports Databricks 13.3 or OSS " + "delta 2.4") +@pytest.mark.parametrize("table_ranges", [(range(10), range(20)), # partial delete of target + (range(5), range(5)), # full delete of target + (range(10), range(20, 30)) # no-op delete + ], ids=idfn) +@pytest.mark.parametrize("use_cdf", [True, False], ids=idfn) +@pytest.mark.parametrize("partition_columns", [None, ["a"], ["b"], ["a", "b"]], ids=idfn) +@pytest.mark.parametrize("num_slices", num_slices_to_test, ids=idfn) +def test_delta_merge_match_delete_only(spark_tmp_path, spark_tmp_table_factory, table_ranges, + use_cdf, partition_columns, num_slices): + do_test_delta_merge_match_delete_only(spark_tmp_path, spark_tmp_table_factory, table_ranges, + use_cdf, partition_columns, num_slices, False, + delta_merge_enabled_conf) + +@allow_non_gpu(*delta_meta_allow) +@delta_lake +@ignore_order +@pytest.mark.skipif(not ((is_databricks_runtime() and is_databricks133_or_later()) or + (not is_databricks_runtime() and spark_version().startswith("3.4"))), + reason="Delta Lake Low Shuffle Merge only supports Databricks 13.3 or OSS " + "delta 2.4") +@pytest.mark.parametrize("use_cdf", [True, False], ids=idfn) +@pytest.mark.parametrize("num_slices", num_slices_to_test, ids=idfn) +def test_delta_merge_standard_upsert(spark_tmp_path, spark_tmp_table_factory, use_cdf, num_slices): + do_test_delta_merge_standard_upsert(spark_tmp_path, spark_tmp_table_factory, use_cdf, + num_slices, False, delta_merge_enabled_conf) + +@allow_non_gpu(*delta_meta_allow) +@delta_lake +@ignore_order +@pytest.mark.skipif(not ((is_databricks_runtime() and is_databricks133_or_later()) or + (not is_databricks_runtime() and spark_version().startswith("3.4"))), + reason="Delta Lake Low Shuffle Merge only supports Databricks 13.3 or OSS " + "delta 2.4") +@pytest.mark.parametrize("use_cdf", [True, False], ids=idfn) +@pytest.mark.parametrize("merge_sql", [ + "MERGE INTO {dest_table} d USING {src_table} s ON d.a == s.a" \ + " WHEN MATCHED AND s.b > 'q' THEN UPDATE SET d.a = s.a / 2, d.b = s.b" \ + " WHEN NOT MATCHED THEN INSERT *", + "MERGE INTO {dest_table} d USING {src_table} s ON d.a == s.a" \ + " WHEN NOT MATCHED AND s.b > 'q' THEN INSERT *", + "MERGE INTO {dest_table} d USING {src_table} s ON d.a == s.a" \ + " WHEN MATCHED AND s.b > 'a' AND s.b < 'g' THEN UPDATE SET d.a = s.a / 2, d.b = s.b" \ + " WHEN MATCHED AND s.b > 'g' AND s.b < 'z' THEN UPDATE SET d.a = s.a / 4, d.b = concat('extra', s.b)" \ + " WHEN NOT MATCHED AND s.b > 'b' AND s.b < 'f' THEN INSERT *" \ + " WHEN NOT MATCHED AND s.b > 'f' AND s.b < 'z' THEN INSERT (b) VALUES ('not here')" ], ids=idfn) +@pytest.mark.parametrize("num_slices", num_slices_to_test, ids=idfn) +def test_delta_merge_upsert_with_condition(spark_tmp_path, spark_tmp_table_factory, use_cdf, merge_sql, num_slices): + do_test_delta_merge_upsert_with_condition(spark_tmp_path, spark_tmp_table_factory, use_cdf, + merge_sql, num_slices, False, + delta_merge_enabled_conf) + +@allow_non_gpu(*delta_meta_allow) +@delta_lake +@ignore_order +@pytest.mark.skipif(not ((is_databricks_runtime() and is_databricks133_or_later()) or + (not is_databricks_runtime() and spark_version().startswith("3.4"))), + reason="Delta Lake Low Shuffle Merge only supports Databricks 13.3 or OSS " + "delta 2.4") +@pytest.mark.parametrize("use_cdf", [True, False], ids=idfn) +@pytest.mark.parametrize("num_slices", num_slices_to_test, ids=idfn) +def test_delta_merge_upsert_with_unmatchable_match_condition(spark_tmp_path, spark_tmp_table_factory, use_cdf, num_slices): + do_test_delta_merge_upsert_with_unmatchable_match_condition(spark_tmp_path, + spark_tmp_table_factory, + use_cdf, + num_slices, + False, + delta_merge_enabled_conf) + +@allow_non_gpu(*delta_meta_allow) +@delta_lake +@ignore_order +@pytest.mark.skipif(not ((is_databricks_runtime() and is_databricks133_or_later()) or + (not is_databricks_runtime() and spark_version().startswith("3.4"))), + reason="Delta Lake Low Shuffle Merge only supports Databricks 13.3 or OSS " + "delta 2.4") +@pytest.mark.parametrize("use_cdf", [True, False], ids=idfn) +def test_delta_merge_update_with_aggregation(spark_tmp_path, spark_tmp_table_factory, use_cdf): + do_test_delta_merge_update_with_aggregation(spark_tmp_path, spark_tmp_table_factory, use_cdf, + delta_merge_enabled_conf) + diff --git a/integration_tests/src/main/python/delta_lake_merge_common.py b/integration_tests/src/main/python/delta_lake_merge_common.py new file mode 100644 index 00000000000..e6e9676625d --- /dev/null +++ b/integration_tests/src/main/python/delta_lake_merge_common.py @@ -0,0 +1,155 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pyspark.sql.functions as f +import string +from pyspark.sql.types import * + +from asserts import * +from data_gen import * +from delta_lake_utils import * +from spark_session import is_databricks_runtime + +# Databricks changes the number of files being written, so we cannot compare logs +num_slices_to_test = [10] if is_databricks_runtime() else [1, 10] + + +def make_df(spark, gen, num_slices): + return three_col_df(spark, gen, SetValuesGen(StringType(), string.ascii_lowercase), + SetValuesGen(StringType(), string.ascii_uppercase), num_slices=num_slices) + + +def delta_sql_merge_test(spark_tmp_path, spark_tmp_table_factory, use_cdf, + src_table_func, dest_table_func, merge_sql, check_func, + partition_columns=None): + data_path = spark_tmp_path + "/DELTA_DATA" + src_table = spark_tmp_table_factory.get() + + def setup_tables(spark): + setup_delta_dest_tables(spark, data_path, dest_table_func, use_cdf, partition_columns) + src_table_func(spark).createOrReplaceTempView(src_table) + + def do_merge(spark, path): + dest_table = spark_tmp_table_factory.get() + read_delta_path(spark, path).createOrReplaceTempView(dest_table) + return spark.sql(merge_sql.format(src_table=src_table, dest_table=dest_table)).collect() + with_cpu_session(setup_tables) + check_func(data_path, do_merge) + + +def assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, + src_table_func, dest_table_func, merge_sql, + compare_logs, partition_columns=None, conf=None): + assert conf is not None, "conf must be set" + + def read_data(spark, path): + read_func = read_delta_path_with_cdf if use_cdf else read_delta_path + df = read_func(spark, path) + return df.sort(df.columns) + + def checker(data_path, do_merge): + cpu_path = data_path + "/CPU" + gpu_path = data_path + "/GPU" + # compare resulting dataframe from the merge operation (some older Spark versions return empty here) + cpu_result = with_cpu_session(lambda spark: do_merge(spark, cpu_path), conf=conf) + gpu_result = with_gpu_session(lambda spark: do_merge(spark, gpu_path), conf=conf) + assert_equal(cpu_result, gpu_result) + # compare merged table data results, read both via CPU to make sure GPU write can be read by CPU + cpu_result = with_cpu_session(lambda spark: read_data(spark, cpu_path).collect(), conf=conf) + gpu_result = with_cpu_session(lambda spark: read_data(spark, gpu_path).collect(), conf=conf) + assert_equal(cpu_result, gpu_result) + # Using partition columns involves sorting, and there's no guarantees on the task + # partitioning due to random sampling. + if compare_logs and not partition_columns: + with_cpu_session(lambda spark: assert_gpu_and_cpu_delta_logs_equivalent(spark, data_path)) + delta_sql_merge_test(spark_tmp_path, spark_tmp_table_factory, use_cdf, + src_table_func, dest_table_func, merge_sql, checker, partition_columns) + + +def do_test_delta_merge_not_match_insert_only(spark_tmp_path, spark_tmp_table_factory, table_ranges, + use_cdf, partition_columns, num_slices, compare_logs, + conf): + src_range, dest_range = table_ranges + src_table_func = lambda spark: make_df(spark, SetValuesGen(IntegerType(), src_range), num_slices) + dest_table_func = lambda spark: make_df(spark, SetValuesGen(IntegerType(), dest_range), num_slices) + merge_sql = "MERGE INTO {dest_table} USING {src_table} ON {dest_table}.a == {src_table}.a" \ + " WHEN NOT MATCHED THEN INSERT *" + assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, + src_table_func, dest_table_func, merge_sql, compare_logs, + partition_columns, conf=conf) + + +def do_test_delta_merge_match_delete_only(spark_tmp_path, spark_tmp_table_factory, table_ranges, + use_cdf, partition_columns, num_slices, compare_logs, + conf): + src_range, dest_range = table_ranges + src_table_func = lambda spark: make_df(spark, SetValuesGen(IntegerType(), src_range), num_slices) + dest_table_func = lambda spark: make_df(spark, SetValuesGen(IntegerType(), dest_range), num_slices) + merge_sql = "MERGE INTO {dest_table} USING {src_table} ON {dest_table}.a == {src_table}.a" \ + " WHEN MATCHED THEN DELETE" + assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, + src_table_func, dest_table_func, merge_sql, compare_logs, + partition_columns, conf=conf) + + +def do_test_delta_merge_standard_upsert(spark_tmp_path, spark_tmp_table_factory, use_cdf, + num_slices, compare_logs, conf): + # Need to eliminate duplicate keys in the source table otherwise update semantics are ambiguous + src_table_func = lambda spark: two_col_df(spark, int_gen, string_gen, num_slices=num_slices).groupBy("a").agg(f.max("b").alias("b")) + dest_table_func = lambda spark: two_col_df(spark, int_gen, string_gen, seed=1, num_slices=num_slices) + merge_sql = "MERGE INTO {dest_table} USING {src_table} ON {dest_table}.a == {src_table}.a" \ + " WHEN MATCHED THEN UPDATE SET * WHEN NOT MATCHED THEN INSERT *" + assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, + src_table_func, dest_table_func, merge_sql, compare_logs, + conf=conf) + + +def do_test_delta_merge_upsert_with_condition(spark_tmp_path, spark_tmp_table_factory, use_cdf, + merge_sql, num_slices, compare_logs, conf): + # Need to eliminate duplicate keys in the source table otherwise update semantics are ambiguous + src_table_func = lambda spark: two_col_df(spark, int_gen, string_gen, num_slices=num_slices).groupBy("a").agg(f.max("b").alias("b")) + dest_table_func = lambda spark: two_col_df(spark, int_gen, string_gen, seed=1, num_slices=num_slices) + assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, + src_table_func, dest_table_func, merge_sql, compare_logs, + conf=conf) + + +def do_test_delta_merge_upsert_with_unmatchable_match_condition(spark_tmp_path, + spark_tmp_table_factory, use_cdf, + num_slices, compare_logs, conf): + # Need to eliminate duplicate keys in the source table otherwise update semantics are ambiguous + src_table_func = lambda spark: two_col_df(spark, int_gen, string_gen, num_slices=num_slices).groupBy("a").agg(f.max("b").alias("b")) + dest_table_func = lambda spark: two_col_df(spark, SetValuesGen(IntegerType(), range(100)), string_gen, seed=1, num_slices=num_slices) + merge_sql = "MERGE INTO {dest_table} USING {src_table} ON {dest_table}.a == {src_table}.a" \ + " WHEN MATCHED AND {dest_table}.a > 100 THEN UPDATE SET *" + assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, + src_table_func, dest_table_func, merge_sql, compare_logs, + conf=conf) + + +def do_test_delta_merge_update_with_aggregation(spark_tmp_path, spark_tmp_table_factory, use_cdf, + conf): + # Need to eliminate duplicate keys in the source table otherwise update semantics are ambiguous + src_table_func = lambda spark: spark.range(10).withColumn("x", f.col("id") + 1) \ + .select(f.col("id"), (f.col("x") + 1).alias("x")) \ + .drop_duplicates(["id"]) \ + .limit(10) + dest_table_func = lambda spark: spark.range(5).withColumn("x", f.col("id") + 1) + merge_sql = "MERGE INTO {dest_table} USING {src_table} ON {dest_table}.id == {src_table}.id" \ + " WHEN MATCHED THEN UPDATE SET {dest_table}.x = {src_table}.x + 2" \ + " WHEN NOT MATCHED AND {src_table}.x < 7 THEN INSERT *" + + assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, + src_table_func, dest_table_func, merge_sql, + compare_logs=False, conf=conf) diff --git a/integration_tests/src/main/python/delta_lake_merge_test.py b/integration_tests/src/main/python/delta_lake_merge_test.py index 0880db16434..5c3bb915ddb 100644 --- a/integration_tests/src/main/python/delta_lake_merge_test.py +++ b/integration_tests/src/main/python/delta_lake_merge_test.py @@ -14,66 +14,17 @@ import pyspark.sql.functions as f import pytest -import string -from asserts import * -from data_gen import * -from delta_lake_utils import * +from delta_lake_merge_common import * from marks import * from pyspark.sql.types import * from spark_session import is_before_spark_320, is_databricks_runtime, spark_version -# Databricks changes the number of files being written, so we cannot compare logs -num_slices_to_test = [10] if is_databricks_runtime() else [1, 10] delta_merge_enabled_conf = copy_and_update(delta_writes_enabled_conf, {"spark.rapids.sql.command.MergeIntoCommand": "true", "spark.rapids.sql.command.MergeIntoCommandEdge": "true"}) -def make_df(spark, gen, num_slices): - return three_col_df(spark, gen, SetValuesGen(StringType(), string.ascii_lowercase), - SetValuesGen(StringType(), string.ascii_uppercase), num_slices=num_slices) - -def delta_sql_merge_test(spark_tmp_path, spark_tmp_table_factory, use_cdf, - src_table_func, dest_table_func, merge_sql, check_func, - partition_columns=None): - data_path = spark_tmp_path + "/DELTA_DATA" - src_table = spark_tmp_table_factory.get() - def setup_tables(spark): - setup_delta_dest_tables(spark, data_path, dest_table_func, use_cdf, partition_columns) - src_table_func(spark).createOrReplaceTempView(src_table) - def do_merge(spark, path): - dest_table = spark_tmp_table_factory.get() - read_delta_path(spark, path).createOrReplaceTempView(dest_table) - return spark.sql(merge_sql.format(src_table=src_table, dest_table=dest_table)).collect() - with_cpu_session(setup_tables) - check_func(data_path, do_merge) - -def assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, - src_table_func, dest_table_func, merge_sql, - compare_logs, partition_columns=None, - conf=delta_merge_enabled_conf): - def read_data(spark, path): - read_func = read_delta_path_with_cdf if use_cdf else read_delta_path - df = read_func(spark, path) - return df.sort(df.columns) - def checker(data_path, do_merge): - cpu_path = data_path + "/CPU" - gpu_path = data_path + "/GPU" - # compare resulting dataframe from the merge operation (some older Spark versions return empty here) - cpu_result = with_cpu_session(lambda spark: do_merge(spark, cpu_path), conf=conf) - gpu_result = with_gpu_session(lambda spark: do_merge(spark, gpu_path), conf=conf) - assert_equal(cpu_result, gpu_result) - # compare merged table data results, read both via CPU to make sure GPU write can be read by CPU - cpu_result = with_cpu_session(lambda spark: read_data(spark, cpu_path).collect(), conf=conf) - gpu_result = with_cpu_session(lambda spark: read_data(spark, gpu_path).collect(), conf=conf) - assert_equal(cpu_result, gpu_result) - # Using partition columns involves sorting, and there's no guarantees on the task - # partitioning due to random sampling. - if compare_logs and not partition_columns: - with_cpu_session(lambda spark: assert_gpu_and_cpu_delta_logs_equivalent(spark, data_path)) - delta_sql_merge_test(spark_tmp_path, spark_tmp_table_factory, use_cdf, - src_table_func, dest_table_func, merge_sql, checker, partition_columns) @allow_non_gpu(delta_write_fallback_allow, *delta_meta_allow) @delta_lake @@ -162,16 +113,9 @@ def test_delta_merge_partial_fallback_via_conf(spark_tmp_path, spark_tmp_table_f @pytest.mark.parametrize("num_slices", num_slices_to_test, ids=idfn) def test_delta_merge_not_match_insert_only(spark_tmp_path, spark_tmp_table_factory, table_ranges, use_cdf, partition_columns, num_slices): - src_range, dest_range = table_ranges - src_table_func = lambda spark: make_df(spark, SetValuesGen(IntegerType(), src_range), num_slices) - dest_table_func = lambda spark: make_df(spark, SetValuesGen(IntegerType(), dest_range), num_slices) - merge_sql = "MERGE INTO {dest_table} USING {src_table} ON {dest_table}.a == {src_table}.a" \ - " WHEN NOT MATCHED THEN INSERT *" - # Non-deterministic input for each task means we can only reliably compare record counts when using only one task - compare_logs = num_slices == 1 - assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, - src_table_func, dest_table_func, merge_sql, compare_logs, - partition_columns) + do_test_delta_merge_not_match_insert_only(spark_tmp_path, spark_tmp_table_factory, + table_ranges, use_cdf, partition_columns, + num_slices, num_slices == 1, delta_merge_enabled_conf) @allow_non_gpu(*delta_meta_allow) @delta_lake @@ -186,16 +130,9 @@ def test_delta_merge_not_match_insert_only(spark_tmp_path, spark_tmp_table_facto @pytest.mark.parametrize("num_slices", num_slices_to_test, ids=idfn) def test_delta_merge_match_delete_only(spark_tmp_path, spark_tmp_table_factory, table_ranges, use_cdf, partition_columns, num_slices): - src_range, dest_range = table_ranges - src_table_func = lambda spark: make_df(spark, SetValuesGen(IntegerType(), src_range), num_slices) - dest_table_func = lambda spark: make_df(spark, SetValuesGen(IntegerType(), dest_range), num_slices) - merge_sql = "MERGE INTO {dest_table} USING {src_table} ON {dest_table}.a == {src_table}.a" \ - " WHEN MATCHED THEN DELETE" - # Non-deterministic input for each task means we can only reliably compare record counts when using only one task - compare_logs = num_slices == 1 - assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, - src_table_func, dest_table_func, merge_sql, compare_logs, - partition_columns) + do_test_delta_merge_match_delete_only(spark_tmp_path, spark_tmp_table_factory, table_ranges, + use_cdf, partition_columns, num_slices, num_slices == 1, + delta_merge_enabled_conf) @allow_non_gpu(*delta_meta_allow) @delta_lake @@ -204,15 +141,9 @@ def test_delta_merge_match_delete_only(spark_tmp_path, spark_tmp_table_factory, @pytest.mark.parametrize("use_cdf", [True, False], ids=idfn) @pytest.mark.parametrize("num_slices", num_slices_to_test, ids=idfn) def test_delta_merge_standard_upsert(spark_tmp_path, spark_tmp_table_factory, use_cdf, num_slices): - # Need to eliminate duplicate keys in the source table otherwise update semantics are ambiguous - src_table_func = lambda spark: two_col_df(spark, int_gen, string_gen, num_slices=num_slices).groupBy("a").agg(f.max("b").alias("b")) - dest_table_func = lambda spark: two_col_df(spark, int_gen, string_gen, seed=1, num_slices=num_slices) - merge_sql = "MERGE INTO {dest_table} USING {src_table} ON {dest_table}.a == {src_table}.a" \ - " WHEN MATCHED THEN UPDATE SET * WHEN NOT MATCHED THEN INSERT *" - # Non-deterministic input for each task means we can only reliably compare record counts when using only one task - compare_logs = num_slices == 1 - assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, - src_table_func, dest_table_func, merge_sql, compare_logs) + do_test_delta_merge_standard_upsert(spark_tmp_path, spark_tmp_table_factory, use_cdf, + num_slices, num_slices == 1, delta_merge_enabled_conf) + @allow_non_gpu(*delta_meta_allow) @delta_lake @@ -232,13 +163,10 @@ def test_delta_merge_standard_upsert(spark_tmp_path, spark_tmp_table_factory, us " WHEN NOT MATCHED AND s.b > 'f' AND s.b < 'z' THEN INSERT (b) VALUES ('not here')" ], ids=idfn) @pytest.mark.parametrize("num_slices", num_slices_to_test, ids=idfn) def test_delta_merge_upsert_with_condition(spark_tmp_path, spark_tmp_table_factory, use_cdf, merge_sql, num_slices): - # Need to eliminate duplicate keys in the source table otherwise update semantics are ambiguous - src_table_func = lambda spark: two_col_df(spark, int_gen, string_gen, num_slices=num_slices).groupBy("a").agg(f.max("b").alias("b")) - dest_table_func = lambda spark: two_col_df(spark, int_gen, string_gen, seed=1, num_slices=num_slices) - # Non-deterministic input for each task means we can only reliably compare record counts when using only one task - compare_logs = num_slices == 1 - assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, - src_table_func, dest_table_func, merge_sql, compare_logs) + do_test_delta_merge_upsert_with_condition(spark_tmp_path, spark_tmp_table_factory, use_cdf, + merge_sql, num_slices, num_slices == 1, + delta_merge_enabled_conf) + @allow_non_gpu(*delta_meta_allow) @delta_lake @@ -247,15 +175,10 @@ def test_delta_merge_upsert_with_condition(spark_tmp_path, spark_tmp_table_facto @pytest.mark.parametrize("use_cdf", [True, False], ids=idfn) @pytest.mark.parametrize("num_slices", num_slices_to_test, ids=idfn) def test_delta_merge_upsert_with_unmatchable_match_condition(spark_tmp_path, spark_tmp_table_factory, use_cdf, num_slices): - # Need to eliminate duplicate keys in the source table otherwise update semantics are ambiguous - src_table_func = lambda spark: two_col_df(spark, int_gen, string_gen, num_slices=num_slices).groupBy("a").agg(f.max("b").alias("b")) - dest_table_func = lambda spark: two_col_df(spark, SetValuesGen(IntegerType(), range(100)), string_gen, seed=1, num_slices=num_slices) - merge_sql = "MERGE INTO {dest_table} USING {src_table} ON {dest_table}.a == {src_table}.a" \ - " WHEN MATCHED AND {dest_table}.a > 100 THEN UPDATE SET *" - # Non-deterministic input for each task means we can only reliably compare record counts when using only one task - compare_logs = num_slices == 1 - assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, - src_table_func, dest_table_func, merge_sql, compare_logs) + do_test_delta_merge_upsert_with_unmatchable_match_condition(spark_tmp_path, + spark_tmp_table_factory, use_cdf, + num_slices, num_slices == 1, + delta_merge_enabled_conf) @allow_non_gpu(*delta_meta_allow) @delta_lake @@ -263,18 +186,8 @@ def test_delta_merge_upsert_with_unmatchable_match_condition(spark_tmp_path, spa @pytest.mark.skipif(is_before_spark_320(), reason="Delta Lake writes are not supported before Spark 3.2.x") @pytest.mark.parametrize("use_cdf", [True, False], ids=idfn) def test_delta_merge_update_with_aggregation(spark_tmp_path, spark_tmp_table_factory, use_cdf): - # Need to eliminate duplicate keys in the source table otherwise update semantics are ambiguous - src_table_func = lambda spark: spark.range(10).withColumn("x", f.col("id") + 1)\ - .select(f.col("id"), (f.col("x") + 1).alias("x"))\ - .drop_duplicates(["id"])\ - .limit(10) - dest_table_func = lambda spark: spark.range(5).withColumn("x", f.col("id") + 1) - merge_sql = "MERGE INTO {dest_table} USING {src_table} ON {dest_table}.id == {src_table}.id" \ - " WHEN MATCHED THEN UPDATE SET {dest_table}.x = {src_table}.x + 2" \ - " WHEN NOT MATCHED AND {src_table}.x < 7 THEN INSERT *" - - assert_delta_sql_merge_collect(spark_tmp_path, spark_tmp_table_factory, use_cdf, - src_table_func, dest_table_func, merge_sql, compare_logs=False) + do_test_delta_merge_update_with_aggregation(spark_tmp_path, spark_tmp_table_factory, use_cdf, + delta_merge_enabled_conf) @allow_non_gpu(*delta_meta_allow) @delta_lake diff --git a/pom.xml b/pom.xml index 06947857521..3ff87c3cb97 100644 --- a/pom.xml +++ b/pom.xml @@ -733,6 +733,7 @@ --> -Xlint:all,-serial,-path,-try,-processing|-Werror 1.16.0 + 1.0.6 ${ucx.baseVersion} true @@ -1016,6 +1017,15 @@ ${alluxio.client.version} provided + + + org.roaringbitmap + RoaringBitmap + ${roaringbitmap.version} + compile + org.scalatest scalatest_${scala.binary.version} diff --git a/scala2.13/aggregator/pom.xml b/scala2.13/aggregator/pom.xml index 1d70c76f037..198b62d5fa6 100644 --- a/scala2.13/aggregator/pom.xml +++ b/scala2.13/aggregator/pom.xml @@ -94,6 +94,10 @@ com.google.flatbuffers ${rapids.shade.package}.com.google.flatbuffers + + org.roaringbitmap + ${rapids.shade.package}.org.roaringbitmap + diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index cbc4aecbd26..e32a64f0529 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -733,6 +733,7 @@ --> -Xlint:all,-serial,-path,-try,-processing|-Werror 1.16.0 + 1.0.6 ${ucx.baseVersion} true @@ -1016,6 +1017,15 @@ ${alluxio.client.version} provided + + + org.roaringbitmap + RoaringBitmap + ${roaringbitmap.version} + compile + org.scalatest scalatest_${scala.binary.version} diff --git a/scala2.13/sql-plugin/pom.xml b/scala2.13/sql-plugin/pom.xml index df3532a3592..eb6f240a3f6 100644 --- a/scala2.13/sql-plugin/pom.xml +++ b/scala2.13/sql-plugin/pom.xml @@ -97,6 +97,10 @@ org.alluxio alluxio-shaded-client + + org.roaringbitmap + RoaringBitmap + org.mockito mockito-core diff --git a/sql-plugin/pom.xml b/sql-plugin/pom.xml index 961e6f08372..08657a9d40b 100644 --- a/sql-plugin/pom.xml +++ b/sql-plugin/pom.xml @@ -97,6 +97,10 @@ org.alluxio alluxio-shaded-client + + org.roaringbitmap + RoaringBitmap + org.mockito mockito-core diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index 8ea1641fb4a..5203e926efa 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -2274,6 +2274,32 @@ val SHUFFLE_COMPRESSION_LZ4_CHUNK_SIZE = conf("spark.rapids.shuffle.compression. .integerConf .createWithDefault(1024) + val DELTA_LOW_SHUFFLE_MERGE_SCATTER_DEL_VECTOR_BATCH_SIZE = + conf("spark.rapids.sql.delta.lowShuffleMerge.deletion.scatter.max.size") + .doc("Option to set max batch size when scattering deletion vector") + .internal() + .integerConf + .createWithDefault(32 * 1024) + + val DELTA_LOW_SHUFFLE_MERGE_DEL_VECTOR_BROADCAST_THRESHOLD = + conf("spark.rapids.sql.delta.lowShuffleMerge.deletionVector.broadcast.threshold") + .doc("Currently we need to broadcast deletion vector to all executors to perform low " + + "shuffle merge. When we detect the deletion vector broadcast size is larger than this " + + "value, we will fallback to normal shuffle merge.") + .bytesConf(ByteUnit.BYTE) + .createWithDefault(20 * 1024 * 1024) + + val ENABLE_DELTA_LOW_SHUFFLE_MERGE = + conf("spark.rapids.sql.delta.lowShuffleMerge.enabled") + .doc("Option to turn on the low shuffle merge for Delta Lake. Currently there are some " + + "limitations for this feature: \n" + + "1. We only support Databricks Runtime 13.3 and Deltalake 2.4. \n" + + s"2. The file scan mode must be set to ${RapidsReaderType.PERFILE} \n" + + "3. The deletion vector size must be smaller than " + + s"${DELTA_LOW_SHUFFLE_MERGE_DEL_VECTOR_BROADCAST_THRESHOLD.key} \n") + .booleanConf + .createWithDefault(false) + private def printSectionHeader(category: String): Unit = println(s"\n### $category") @@ -3083,6 +3109,8 @@ class RapidsConf(conf: Map[String, String]) extends Logging { lazy val testGetJsonObjectSaveRows: Int = get(TEST_GET_JSON_OBJECT_SAVE_ROWS) + lazy val isDeltaLowShuffleMergeEnabled: Boolean = get(ENABLE_DELTA_LOW_SHUFFLE_MERGE) + private val optimizerDefaults = Map( // this is not accurate because CPU projections do have a cost due to appending values // to each row that is produced, but this needs to be a really small number because From 4b449034f2a0105c687646176590b349f9901ea7 Mon Sep 17 00:00:00 2001 From: Liangcai Li Date: Mon, 24 Jun 2024 09:32:03 +0800 Subject: [PATCH 40/79] Support bucketing write for GPU (#10957) This PR adds the GPU support for the bucketing write. - React the code of the dynamic partition single writer and concurrent writer to try to reuse the code as much as possible, and then add in the bucketing write logic for both of them. - Update the bucket check during the plan overriding for the write commands, including InsertIntoHadoopFsRelationCommand, CreateDataSourceTableAsSelectCommand, InsertIntoHiveTable, CreateHiveTableAsSelectCommand. - From 330, Spark also supports HiveHash to generate the bucket IDs, in addition to Murmur3Hash. So the shim object GpuBucketingUtils is introduced to handle the shim things. - This change also adds two functions (tagForHiveBucketingWrite and tagForBucketing) to do the overriding check for the two hashing functions separately. And the Hive write nodes will fall back to CPU when HiveHash is chosen, because HiveHash is not supported on GPU. --------- Signed-off-by: Firestarman --- integration_tests/src/main/python/asserts.py | 6 +- .../src/main/python/orc_write_test.py | 48 +- .../src/main/python/parquet_write_test.py | 79 +- .../rapids/GpuHashPartitioningBase.scala | 8 +- .../nvidia/spark/rapids/GpuOverrides.scala | 7 +- .../sql/hive/rapids/GpuHiveFileFormat.scala | 6 +- .../sql/rapids/GpuFileFormatDataWriter.scala | 1112 +++++++---------- ...aSourceTableAsSelectCommandMetaShims.scala | 8 +- ...dCreateHiveTableAsSelectCommandShims.scala | 5 +- .../shims/spark311/GpuBucketingUtils.scala | 77 ++ .../GpuCreateHiveTableAsSelectCommand.scala | 9 +- .../rapids/shims/GpuInsertIntoHiveTable.scala | 5 +- .../sql/rapids/GpuFileFormatWriter.scala | 15 +- .../shims/spark330/GpuBucketingUtils.scala | 88 ++ ...aSourceTableAsSelectCommandMetaShims.scala | 12 +- .../rapids/shims/GpuInsertIntoHiveTable.scala | 7 +- ...dCreateHiveTableAsSelectCommandShims.scala | 6 +- .../sql/rapids/GpuFileFormatWriter.scala | 15 +- .../rapids/GpuFileFormatDataWriterSuite.scala | 132 +- 19 files changed, 896 insertions(+), 749 deletions(-) create mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/spark311/GpuBucketingUtils.scala create mode 100644 sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/GpuBucketingUtils.scala diff --git a/integration_tests/src/main/python/asserts.py b/integration_tests/src/main/python/asserts.py index 32416612d26..b861e89b726 100644 --- a/integration_tests/src/main/python/asserts.py +++ b/integration_tests/src/main/python/asserts.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -101,6 +101,10 @@ def _assert_equal(cpu, gpu, float_check, path): else: assert False, "Found unexpected type {} at {}".format(t, path) +def assert_equal_with_local_sort(cpu, gpu): + _sort_locally(cpu, gpu) + assert_equal(cpu, gpu) + def assert_equal(cpu, gpu): """Verify that the result from the CPU and the GPU are equal""" try: diff --git a/integration_tests/src/main/python/orc_write_test.py b/integration_tests/src/main/python/orc_write_test.py index 8d3013cbe8b..5b5c7b786b6 100644 --- a/integration_tests/src/main/python/orc_write_test.py +++ b/integration_tests/src/main/python/orc_write_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -209,7 +209,7 @@ def test_write_sql_save_table(spark_tmp_path, orc_gens, ts_type, orc_impl, spark @pytest.mark.parametrize('codec', ['zlib', 'lzo']) def test_orc_write_compression_fallback(spark_tmp_path, codec, spark_tmp_table_factory): gen = TimestampGen() - data_path = spark_tmp_path + '/PARQUET_DATA' + data_path = spark_tmp_path + '/ORC_DATA' all_confs={'spark.sql.orc.compression.codec': codec, 'spark.rapids.sql.format.orc.write.enabled': True} assert_gpu_fallback_write( lambda spark, path: unary_op_df(spark, gen).coalesce(1).write.format("orc").mode('overwrite').option("path", path).saveAsTable(spark_tmp_table_factory.get()), @@ -218,17 +218,45 @@ def test_orc_write_compression_fallback(spark_tmp_path, codec, spark_tmp_table_f 'DataWritingCommandExec', conf=all_confs) -@ignore_order -@allow_non_gpu('DataWritingCommandExec,ExecutedCommandExec,WriteFilesExec') -def test_buckets_write_fallback(spark_tmp_path, spark_tmp_table_factory): +@ignore_order(local=True) +def test_buckets_write_round_trip(spark_tmp_path, spark_tmp_table_factory): data_path = spark_tmp_path + '/ORC_DATA' + gen_list = [["id", int_gen], ["data", long_gen]] + assert_gpu_and_cpu_writes_are_equal_collect( + lambda spark, path: gen_df(spark, gen_list).selectExpr("id % 100 as b_id", "data").write + .bucketBy(4, "b_id").format('orc').mode('overwrite').option("path", path) + .saveAsTable(spark_tmp_table_factory.get()), + lambda spark, path: spark.read.orc(path), + data_path, + conf={'spark.rapids.sql.format.orc.write.enabled': True}) + +@ignore_order(local=True) +@allow_non_gpu('DataWritingCommandExec,ExecutedCommandExec,WriteFilesExec, SortExec') +def test_buckets_write_fallback_unsupported_types(spark_tmp_path, spark_tmp_table_factory): + data_path = spark_tmp_path + '/ORC_DATA' + gen_list = [["id", binary_gen], ["data", long_gen]] assert_gpu_fallback_write( - lambda spark, path: spark.range(10e4).write.bucketBy(4, "id").sortBy("id").format('orc').mode('overwrite').option("path", path).saveAsTable(spark_tmp_table_factory.get()), - lambda spark, path: spark.read.orc(path), - data_path, - 'DataWritingCommandExec', - conf = {'spark.rapids.sql.format.orc.write.enabled': True}) + lambda spark, path: gen_df(spark, gen_list).selectExpr("id as b_id", "data").write + .bucketBy(4, "b_id").format('orc').mode('overwrite').option("path", path) + .saveAsTable(spark_tmp_table_factory.get()), + lambda spark, path: spark.read.orc(path), + data_path, + 'DataWritingCommandExec', + conf={'spark.rapids.sql.format.orc.write.enabled': True}) +@ignore_order(local=True) +def test_partitions_and_buckets_write_round_trip(spark_tmp_path, spark_tmp_table_factory): + data_path = spark_tmp_path + '/ORC_DATA' + gen_list = [["id", int_gen], ["data", long_gen]] + assert_gpu_and_cpu_writes_are_equal_collect( + lambda spark, path: gen_df(spark, gen_list) + .selectExpr("id % 5 as b_id", "id % 10 as p_id", "data").write + .partitionBy("p_id") + .bucketBy(4, "b_id").format('orc').mode('overwrite').option("path", path) + .saveAsTable(spark_tmp_table_factory.get()), + lambda spark, path: spark.read.orc(path), + data_path, + conf={'spark.rapids.sql.format.orc.write.enabled': True}) @ignore_order @allow_non_gpu('DataWritingCommandExec,ExecutedCommandExec,WriteFilesExec') diff --git a/integration_tests/src/main/python/parquet_write_test.py b/integration_tests/src/main/python/parquet_write_test.py index 38dab9e84a4..805a0b8137c 100644 --- a/integration_tests/src/main/python/parquet_write_test.py +++ b/integration_tests/src/main/python/parquet_write_test.py @@ -409,16 +409,81 @@ def test_parquet_writeLegacyFormat_fallback(spark_tmp_path, spark_tmp_table_fact 'DataWritingCommandExec', conf=all_confs) -@ignore_order -@allow_non_gpu('DataWritingCommandExec,ExecutedCommandExec,WriteFilesExec') -def test_buckets_write_fallback(spark_tmp_path, spark_tmp_table_factory): +@ignore_order(local=True) +def test_buckets_write_round_trip(spark_tmp_path, spark_tmp_table_factory): + data_path = spark_tmp_path + '/PARQUET_DATA' + gen_list = [["id", int_gen], ["data", long_gen]] + assert_gpu_and_cpu_writes_are_equal_collect( + lambda spark, path: gen_df(spark, gen_list).selectExpr("id % 100 as b_id", "data").write + .bucketBy(4, "b_id").format('parquet').mode('overwrite').option("path", path) + .saveAsTable(spark_tmp_table_factory.get()), + lambda spark, path: spark.read.parquet(path), + data_path, + conf=writer_confs) + + +def test_buckets_write_correctness(spark_tmp_path, spark_tmp_table_factory): + cpu_path = spark_tmp_path + '/PARQUET_DATA/CPU' + gpu_path = spark_tmp_path + '/PARQUET_DATA/GPU' + gen_list = [["id", int_gen], ["data", long_gen]] + num_buckets = 4 + + def do_bucketing_write(spark, path): + df = gen_df(spark, gen_list).selectExpr("id % 100 as b_id", "data") + df.write.bucketBy(num_buckets, "b_id").format('parquet').mode('overwrite') \ + .option("path", path).saveAsTable(spark_tmp_table_factory.get()) + + def read_single_bucket(path, bucket_id): + # Bucket Id string format: f"_$id%05d" + ".c$fileCounter%03d". + # fileCounter is always 0 in this test. For example '_00002.c000' is for + # bucket id being 2. + # We leverage this bucket segment in the file path to filter rows belong + # to a bucket. + bucket_segment = '_' + "{}".format(bucket_id).rjust(5, '0') + '.c000' + return with_cpu_session( + lambda spark: spark.read.parquet(path) + .withColumn('file_name', f.input_file_name()) + .filter(f.col('file_name').contains(bucket_segment)) + .selectExpr('b_id', 'data') # need to drop the file_name column for comparison. + .collect()) + + with_cpu_session(lambda spark: do_bucketing_write(spark, cpu_path), writer_confs) + with_gpu_session(lambda spark: do_bucketing_write(spark, gpu_path), writer_confs) + cur_bucket_id = 0 + while cur_bucket_id < num_buckets: + # Verify the result bucket by bucket + ret_cpu = read_single_bucket(cpu_path, cur_bucket_id) + ret_gpu = read_single_bucket(gpu_path, cur_bucket_id) + assert_equal_with_local_sort(ret_cpu, ret_gpu) + cur_bucket_id += 1 + +@ignore_order(local=True) +@allow_non_gpu('DataWritingCommandExec,ExecutedCommandExec,WriteFilesExec, SortExec') +def test_buckets_write_fallback_unsupported_types(spark_tmp_path, spark_tmp_table_factory): data_path = spark_tmp_path + '/PARQUET_DATA' + gen_list = [["id", binary_gen], ["data", long_gen]] assert_gpu_fallback_write( - lambda spark, path: spark.range(10e4).write.bucketBy(4, "id").sortBy("id").format('parquet').mode('overwrite').option("path", path).saveAsTable(spark_tmp_table_factory.get()), - lambda spark, path: spark.read.parquet(path), - data_path, - 'DataWritingCommandExec') + lambda spark, path: gen_df(spark, gen_list).selectExpr("id as b_id", "data").write + .bucketBy(4, "b_id").format('parquet').mode('overwrite').option("path", path) + .saveAsTable(spark_tmp_table_factory.get()), + lambda spark, path: spark.read.parquet(path), + data_path, + 'DataWritingCommandExec', + conf=writer_confs) +@ignore_order(local=True) +def test_partitions_and_buckets_write_round_trip(spark_tmp_path, spark_tmp_table_factory): + data_path = spark_tmp_path + '/PARQUET_DATA' + gen_list = [["id", int_gen], ["data", long_gen]] + assert_gpu_and_cpu_writes_are_equal_collect( + lambda spark, path: gen_df(spark, gen_list) + .selectExpr("id % 5 as b_id", "id % 10 as p_id", "data").write + .partitionBy("p_id") + .bucketBy(4, "b_id").format('parquet').mode('overwrite').option("path", path) + .saveAsTable(spark_tmp_table_factory.get()), + lambda spark, path: spark.read.parquet(path), + data_path, + conf=writer_confs) @ignore_order @allow_non_gpu('DataWritingCommandExec,ExecutedCommandExec,WriteFilesExec') diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashPartitioningBase.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashPartitioningBase.scala index b17b2782e90..baa009d0669 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashPartitioningBase.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuHashPartitioningBase.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ import com.nvidia.spark.rapids.Arm.withResource import com.nvidia.spark.rapids.shims.ShimExpression import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.rapids.GpuMurmur3Hash +import org.apache.spark.sql.rapids.{GpuMurmur3Hash, GpuPmod} import org.apache.spark.sql.types.{DataType, IntegerType} import org.apache.spark.sql.vectorized.ColumnarBatch @@ -59,6 +59,10 @@ abstract class GpuHashPartitioningBase(expressions: Seq[Expression], numPartitio sliceInternalGpuOrCpuAndClose(numRows, partitionIndexes, partitionColumns) } } + + def partitionIdExpression: GpuExpression = GpuPmod( + GpuMurmur3Hash(expressions, GpuHashPartitioningBase.DEFAULT_HASH_SEED), + GpuLiteral(numPartitions)) } object GpuHashPartitioningBase { diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 295480d24cc..9e26cf751f4 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -322,10 +322,11 @@ final class InsertIntoHadoopFsRelationCommandMeta( private var fileFormat: Option[ColumnarFileFormat] = None override def tagSelfForGpuInternal(): Unit = { - if (cmd.bucketSpec.isDefined) { - willNotWorkOnGpu("bucketing is not supported") + if (GpuBucketingUtils.isHiveHashBucketing(cmd.options)) { + GpuBucketingUtils.tagForHiveBucketingWrite(this, cmd.bucketSpec, cmd.outputColumns, false) + } else { + BucketIdMetaUtils.tagForBucketingWrite(this, cmd.bucketSpec, cmd.outputColumns) } - val spark = SparkSession.active val formatCls = cmd.fileFormat.getClass fileFormat = if (formatCls == classOf[CSVFileFormat]) { diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveFileFormat.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveFileFormat.scala index 21437a64481..69189b2600c 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveFileFormat.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveFileFormat.scala @@ -24,6 +24,7 @@ import com.google.common.base.Charsets import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.Arm.withResource import com.nvidia.spark.rapids.jni.CastStrings +import com.nvidia.spark.rapids.shims.GpuBucketingUtils import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext} import org.apache.spark.internal.Logging @@ -43,9 +44,8 @@ object GpuHiveFileFormat extends Logging { def tagGpuSupport(meta: GpuInsertIntoHiveTableMeta): Option[ColumnarFileFormat] = { val insertCmd = meta.wrapped // Bucketing write - if (insertCmd.table.bucketSpec.isDefined) { - meta.willNotWorkOnGpu("bucketed tables are not supported yet") - } + GpuBucketingUtils.tagForHiveBucketingWrite(meta, insertCmd.table.bucketSpec, + insertCmd.outputColumns, false) // Infer the file format from the serde string, similar as what Spark does in // RelationConversions for Hive. diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileFormatDataWriter.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileFormatDataWriter.scala index 4ceac365314..939a421e0b9 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileFormatDataWriter.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileFormatDataWriter.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,30 +17,30 @@ package org.apache.spark.sql.rapids import scala.collection.mutable -import scala.collection.mutable.{ArrayBuffer, ListBuffer} +import scala.collection.mutable.ListBuffer +import scala.util.hashing.{MurmurHash3 => ScalaMurmur3Hash} -import ai.rapids.cudf.{ColumnVector, OrderByArg, Table} +import ai.rapids.cudf.{OrderByArg, Table} import com.nvidia.spark.TimingUtils import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.Arm.{closeOnExcept, withResource} import com.nvidia.spark.rapids.RapidsPluginImplicits._ import com.nvidia.spark.rapids.RmmRapidsRetryIterator.withRetryNoSplit -import com.nvidia.spark.rapids.ScalableTaskCompletion.onTaskCompletion import com.nvidia.spark.rapids.shims.GpuFileFormatDataWriterShim import org.apache.hadoop.fs.Path import org.apache.hadoop.mapreduce.TaskAttemptContext -import org.apache.spark.TaskContext +import org.apache.spark.internal.Logging import org.apache.spark.internal.io.FileCommitProtocol import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.catalog.{BucketSpec, ExternalCatalogUtils} import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec -import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils -import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, AttributeSet, Cast, Concat, Expression, Literal, NullsFirst, ScalaUDF, SortOrder, UnsafeProjection} +import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, AttributeSet, Cast, Concat, Expression, Literal, Murmur3Hash, NullsFirst, ScalaUDF, UnsafeProjection} import org.apache.spark.sql.connector.write.DataWriter import org.apache.spark.sql.execution.datasources.{BucketingUtils, PartitioningUtils, WriteTaskResult} -import org.apache.spark.sql.rapids.GpuFileFormatDataWriter.{shouldSplitToFitMaxRecordsPerFile, splitToFitMaxRecordsAndClose} +import org.apache.spark.sql.rapids.GpuFileFormatDataWriter._ import org.apache.spark.sql.rapids.GpuFileFormatWriter.GpuConcurrentOutputWriterSpec -import org.apache.spark.sql.types.{DataType, StringType} +import org.apache.spark.sql.types._ import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.SerializableConfiguration @@ -50,7 +50,7 @@ object GpuFileFormatDataWriter { } def shouldSplitToFitMaxRecordsPerFile( - maxRecordsPerFile: Long, recordsInFile: Long, numRowsInBatch: Long) = { + maxRecordsPerFile: Long, recordsInFile: Long, numRowsInBatch: Long): Boolean = { maxRecordsPerFile > 0 && (recordsInFile + numRowsInBatch) > maxRecordsPerFile } @@ -88,13 +88,8 @@ object GpuFileFormatDataWriter { maxRecordsPerFile: Long, recordsInFile: Long): Array[SpillableColumnarBatch] = { val (types, splitIndexes) = closeOnExcept(batch) { _ => - val types = GpuColumnVector.extractTypes(batch) - val splitIndexes = - getSplitIndexes( - maxRecordsPerFile, - recordsInFile, - batch.numRows()) - (types, splitIndexes) + val splitIndexes = getSplitIndexes(maxRecordsPerFile, recordsInFile, batch.numRows()) + (GpuColumnVector.extractTypes(batch), splitIndexes) } if (splitIndexes.isEmpty) { // this should never happen, as `splitToFitMaxRecordsAndClose` is called when @@ -124,6 +119,31 @@ abstract class GpuFileFormatDataWriter( description: GpuWriteJobDescription, taskAttemptContext: TaskAttemptContext, committer: FileCommitProtocol) extends DataWriter[ColumnarBatch] { + + protected class WriterAndStatus { + var writer: ColumnarOutputWriter = _ + + /** Number of records in current file. */ + var recordsInFile: Long = 0 + + /** + * File counter for writing current partition or bucket. For same partition or bucket, + * we may have more than one file, due to number of records limit per file. + */ + var fileCounter: Int = 0 + + final def release(): Unit = { + if (writer != null) { + try { + writer.close() + statsTrackers.foreach(_.closeFile(writer.path())) + } finally { + writer = null + } + } + } + } + /** * Max number of files a single task writes out due to file size. In most cases the number of * files written should be very small. This is just a safe guard to protect some really bad @@ -131,28 +151,26 @@ abstract class GpuFileFormatDataWriter( */ protected val MAX_FILE_COUNTER: Int = 1000 * 1000 protected val updatedPartitions: mutable.Set[String] = mutable.Set[String]() - protected var currentWriter: ColumnarOutputWriter = _ + protected var currentWriterStatus: WriterAndStatus = new WriterAndStatus() /** Trackers for computing various statistics on the data as it's being written out. */ protected val statsTrackers: Seq[ColumnarWriteTaskStatsTracker] = description.statsTrackers.map(_.newTaskInstance()) - /** Release resources of `currentWriter`. */ - protected def releaseCurrentWriter(): Unit = { - if (currentWriter != null) { - try { - currentWriter.close() - statsTrackers.foreach(_.closeFile(currentWriter.path())) - } finally { - currentWriter = null - } - } + /** Release resources of a WriterStatus. */ + protected final def releaseOutWriter(status: WriterAndStatus): Unit = { + status.release() + } + + protected final def writeUpdateMetricsAndClose(scb: SpillableColumnarBatch, + writerStatus: WriterAndStatus): Unit = { + writerStatus.recordsInFile += writerStatus.writer.writeSpillableAndClose(scb, statsTrackers) } /** Release all resources. Public for testing */ def releaseResources(): Unit = { - // Call `releaseCurrentWriter()` by default, as this is the only resource to be released. - releaseCurrentWriter() + // Release current writer by default, as this is the only resource to be released. + releaseOutWriter(currentWriterStatus) } /** Write an iterator of column batch. */ @@ -211,8 +229,6 @@ class GpuSingleDirectoryDataWriter( taskAttemptContext: TaskAttemptContext, committer: FileCommitProtocol) extends GpuFileFormatDataWriter(description, taskAttemptContext, committer) { - private var fileCounter: Int = _ - private var recordsInFile: Long = _ // Initialize currentWriter and statsTrackers newOutputWriter() @@ -220,7 +236,8 @@ class GpuSingleDirectoryDataWriter( "msg=method newTaskTempFile in class FileCommitProtocol is deprecated" ) private def newOutputWriter(): Unit = { - recordsInFile = 0 + currentWriterStatus.recordsInFile = 0 + val fileCounter = currentWriterStatus.fileCounter releaseResources() val ext = description.outputWriterFactory.getFileExtension(taskAttemptContext) @@ -229,7 +246,7 @@ class GpuSingleDirectoryDataWriter( None, f"-c$fileCounter%03d" + ext) - currentWriter = description.outputWriterFactory.newInstance( + currentWriterStatus.writer = description.outputWriterFactory.newInstance( path = currentPath, dataSchema = description.dataColumns.toStructType, context = taskAttemptContext) @@ -237,32 +254,30 @@ class GpuSingleDirectoryDataWriter( statsTrackers.foreach(_.newFile(currentPath)) } - private def writeUpdateMetricsAndClose(scb: SpillableColumnarBatch): Unit = { - recordsInFile += currentWriter.writeSpillableAndClose(scb, statsTrackers) - } - override def write(batch: ColumnarBatch): Unit = { val maxRecordsPerFile = description.maxRecordsPerFile + val recordsInFile = currentWriterStatus.recordsInFile if (!shouldSplitToFitMaxRecordsPerFile( maxRecordsPerFile, recordsInFile, batch.numRows())) { writeUpdateMetricsAndClose( - SpillableColumnarBatch(batch, SpillPriorities.ACTIVE_ON_DECK_PRIORITY)) + SpillableColumnarBatch(batch, SpillPriorities.ACTIVE_ON_DECK_PRIORITY), + currentWriterStatus) } else { val partBatches = splitToFitMaxRecordsAndClose( batch, maxRecordsPerFile, recordsInFile) - var needNewWriter = recordsInFile >= maxRecordsPerFile + val needNewWriterForFirstPart = recordsInFile >= maxRecordsPerFile closeOnExcept(partBatches) { _ => partBatches.zipWithIndex.foreach { case (partBatch, partIx) => - if (needNewWriter) { - fileCounter += 1 + if (partIx > 0 || needNewWriterForFirstPart) { + currentWriterStatus.fileCounter += 1 + val fileCounter = currentWriterStatus.fileCounter assert(fileCounter <= MAX_FILE_COUNTER, s"File counter $fileCounter is beyond max value $MAX_FILE_COUNTER") newOutputWriter() } // null out the entry so that we don't double close partBatches(partIx) = null - writeUpdateMetricsAndClose(partBatch) - needNewWriter = true + writeUpdateMetricsAndClose(partBatch, currentWriterStatus) } } } @@ -280,35 +295,44 @@ class GpuDynamicPartitionDataSingleWriter( taskAttemptContext: TaskAttemptContext, committer: FileCommitProtocol) extends GpuFileFormatDataWriter(description, taskAttemptContext, committer) { + /** Wrapper class to index a unique concurrent output writer. */ + protected class WriterIndex( + var partitionPath: Option[String], + var bucketId: Option[Int]) extends Product2[Option[String], Option[Int]] { - /** Wrapper class for status of a unique single output writer. */ - protected class WriterStatus( - // output writer - var outputWriter: ColumnarOutputWriter, + override def hashCode(): Int = ScalaMurmur3Hash.productHash(this) - /** Number of records in current file. */ - var recordsInFile: Long = 0, + override def equals(obj: Any): Boolean = { + if (obj.isInstanceOf[WriterIndex]) { + val otherWI = obj.asInstanceOf[WriterIndex] + partitionPath == otherWI.partitionPath && bucketId == otherWI.bucketId + } else { + false + } + } - /** - * File counter for writing current partition or bucket. For same partition or bucket, - * we may have more than one file, due to number of records limit per file. - */ - var fileCounter: Int = 0 - ) + override def _1: Option[String] = partitionPath + override def _2: Option[Int] = bucketId + override def canEqual(that: Any): Boolean = that.isInstanceOf[WriterIndex] + } - /** Wrapper class for status and caches of a unique concurrent output writer. - * Used by `GpuDynamicPartitionDataConcurrentWriter` + /** + * A case class to hold the batch, the optional partition path and the optional bucket + * ID for a split group. All the rows in the batch belong to the group defined by the + * partition path and the bucket ID. */ - class WriterStatusWithCaches( - // writer status - var writerStatus: WriterStatus, - - // caches for this partition or writer - val tableCaches: ListBuffer[SpillableColumnarBatch] = ListBuffer(), - - // current device bytes for the above caches - var deviceBytes: Long = 0 - ) + private case class SplitPack(split: SpillableColumnarBatch, path: Option[String], + bucketId: Option[Int]) extends AutoCloseable { + override def close(): Unit = { + split.safeClose() + } + } + /** + * The index for current writer. Intentionally make the index mutable and reusable. + * Avoid JVM GC issue when many short-living `WriterIndex` objects are created + * if switching between concurrent writers frequently. + */ + private val currentWriterId: WriterIndex = new WriterIndex(None, None) /** Flag saying whether or not the data to be written out is partitioned. */ protected val isPartitioned: Boolean = description.partitionColumns.nonEmpty @@ -316,25 +340,17 @@ class GpuDynamicPartitionDataSingleWriter( /** Flag saying whether or not the data to be written out is bucketed. */ protected val isBucketed: Boolean = description.bucketSpec.isDefined - private var currentPartPath: String = "" - - private var currentWriterStatus: WriterStatus = _ - - // All data is sorted ascending with default null ordering - private val nullsSmallest = Ascending.defaultNullOrdering == NullsFirst - - if (isBucketed) { - throw new UnsupportedOperationException("Bucketing is not supported on the GPU yet.") - } - assert(isPartitioned || isBucketed, s"""GpuDynamicPartitionWriteTask should be used for writing out data that's either |partitioned or bucketed. In this case neither is true. |GpuWriteJobDescription: $description """.stripMargin) + // All data is sorted ascending with default null ordering + private val nullsSmallest = Ascending.defaultNullOrdering == NullsFirst + /** Extracts the partition values out of an input batch. */ - protected lazy val getPartitionColumnsAsBatch: ColumnarBatch => ColumnarBatch = { + private lazy val getPartitionColumnsAsBatch: ColumnarBatch => ColumnarBatch = { val expressions = GpuBindReferences.bindGpuReferences( description.partitionColumns, description.allColumns) @@ -343,20 +359,9 @@ class GpuDynamicPartitionDataSingleWriter( } } - /** Extracts the output values of an input batch. */ - private lazy val getOutputColumnsAsBatch: ColumnarBatch => ColumnarBatch= { + private lazy val getBucketIdColumnAsBatch: ColumnarBatch => ColumnarBatch = { val expressions = GpuBindReferences.bindGpuReferences( - description.dataColumns, - description.allColumns) - cb => { - GpuProjectExec.project(cb, expressions) - } - } - - /** Extracts the output values of an input batch. */ - protected lazy val getOutputCb: ColumnarBatch => ColumnarBatch = { - val expressions = GpuBindReferences.bindGpuReferences( - description.dataColumns, + Seq(description.bucketSpec.get.bucketIdExpression), description.allColumns) cb => { GpuProjectExec.project(cb, expressions) @@ -379,62 +384,58 @@ class GpuDynamicPartitionDataSingleWriter( /** Evaluates the `partitionPathExpression` above on a row of `partitionValues` and returns * the partition string. */ - protected lazy val getPartitionPath: InternalRow => String = { + private lazy val getPartitionPath: InternalRow => String = { val proj = UnsafeProjection.create(Seq(partitionPathExpression), description.partitionColumns) row => proj(row).getString(0) } - /** Release resources of writer. */ - private def releaseWriter(writer: ColumnarOutputWriter): Unit = { - if (writer != null) { - val path = writer.path() - writer.close() - statsTrackers.foreach(_.closeFile(path)) + /** Extracts the output values of an input batch. */ + protected lazy val getDataColumnsAsBatch: ColumnarBatch => ColumnarBatch = { + val expressions = GpuBindReferences.bindGpuReferences( + description.dataColumns, + description.allColumns) + cb => { + GpuProjectExec.project(cb, expressions) } } - /** - * Opens a new OutputWriter given a partition key and/or a bucket id. - * If bucket id is specified, we will append it to the end of the file name, but before the - * file extension, e.g. part-r-00009-ea518ad4-455a-4431-b471-d24e03814677-00002.gz.parquet - * - * @param partDir the partition directory - * @param bucketId the bucket which all tuples being written by this OutputWriter belong to, - * currently does not support `bucketId`, it's always None - * @param fileCounter integer indicating the number of files to be written to `partDir` - */ - @scala.annotation.nowarn( - "msg=method newTaskTempFile.* in class FileCommitProtocol is deprecated" - ) - def newWriter( - partDir: String, - bucketId: Option[Int], // Currently it's always None - fileCounter: Int - ): ColumnarOutputWriter = { - updatedPartitions.add(partDir) - // Currently will be empty - val bucketIdStr = bucketId.map(BucketingUtils.bucketIdToString).getOrElse("") - - // This must be in a form that matches our bucketing format. See BucketingUtils. - val ext = f"$bucketIdStr.c$fileCounter%03d" + - description.outputWriterFactory.getFileExtension(taskAttemptContext) - - val customPath = description.customPartitionLocations - .get(PartitioningUtils.parsePathFragment(partDir)) + protected def getKeysBatch(cb: ColumnarBatch): ColumnarBatch = { + val keysBatch = withResource(getPartitionColumnsAsBatch(cb)) { partCb => + if (isBucketed) { + withResource(getBucketIdColumnAsBatch(cb)) { bucketIdCb => + GpuColumnVector.combineColumns(partCb, bucketIdCb) + } + } else { + GpuColumnVector.incRefCounts(partCb) + } + } + require(keysBatch.numCols() > 0, "No sort key is specified") + keysBatch + } - val currentPath = if (customPath.isDefined) { - committer.newTaskTempFileAbsPath(taskAttemptContext, customPath.get, ext) + protected def genGetBucketIdFunc(keyHostCb: ColumnarBatch): Int => Option[Int] = { + if (isBucketed) { + // The last column is the bucket id column + val bucketIdCol = keyHostCb.column(keyHostCb.numCols() - 1) + i => Some(bucketIdCol.getInt(i)) } else { - committer.newTaskTempFile(taskAttemptContext, Option(partDir), ext) + _ => None } + } - val newWriter = description.outputWriterFactory.newInstance( - path = currentPath, - dataSchema = description.dataColumns.toStructType, - context = taskAttemptContext) - - statsTrackers.foreach(_.newFile(currentPath)) - newWriter + protected def genGetPartitionPathFunc(keyHostCb: ColumnarBatch): Int => Option[String] = { + if (isPartitioned) { + // Use the existing code to convert each row into a path. It would be nice to do this + // on the GPU, but the data should be small and there are things we cannot easily + // support on the GPU right now + import scala.collection.JavaConverters._ + val partCols = description.partitionColumns.indices.map(keyHostCb.column) + val iter = new ColumnarBatch(partCols.toArray, keyHostCb.numRows()).rowIterator() + .asScala.map(getPartitionPath) + _ => Some(iter.next) + } else { + _ => None + } } // distinct value sorted the same way the input data is sorted. @@ -461,282 +462,195 @@ class GpuDynamicPartitionDataSingleWriter( } } - override def write(batch: ColumnarBatch): Unit = { - // this single writer always passes `cachesMap` as None - write(batch, cachesMap = None) - } - - private case class SplitAndPath(var split: SpillableColumnarBatch, path: String) - extends AutoCloseable { - override def close(): Unit = { - split.safeClose() - split = null - } - } - /** - * Split a batch according to the sorted keys (partitions). Returns a tuple with an - * array of the splits as `ContiguousTable`'s, and an array of paths to use to - * write each partition. + * Split a batch according to the sorted keys (partitions + bucket ids). + * Returns a tuple with an array of the splits as `ContiguousTable`'s, an array of + * paths and bucket ids to use to write each partition and(or) bucket file. */ - private def splitBatchByKeyAndClose( - batch: ColumnarBatch, - partDataTypes: Array[DataType]): Array[SplitAndPath] = { - val (outputColumnsBatch, partitionColumnsBatch) = withResource(batch) { _ => - closeOnExcept(getOutputColumnsAsBatch(batch)) { outputColumnsBatch => - closeOnExcept(getPartitionColumnsAsBatch(batch)) { partitionColumnsBatch => - (outputColumnsBatch, partitionColumnsBatch) - } + private def splitBatchByKeyAndClose(batch: ColumnarBatch): Array[SplitPack] = { + val (keysCb, dataCb) = withResource(batch) { _ => + closeOnExcept(getDataColumnsAsBatch(batch)) { data => + (getKeysBatch(batch), data) } } - val (cbKeys, partitionIndexes) = closeOnExcept(outputColumnsBatch) { _ => - val partitionColumnsTbl = withResource(partitionColumnsBatch) { _ => - GpuColumnVector.from(partitionColumnsBatch) - } - withResource(partitionColumnsTbl) { _ => - withResource(distinctAndSort(partitionColumnsTbl)) { distinctKeysTbl => - val partitionIndexes = splitIndexes(partitionColumnsTbl, distinctKeysTbl) - val cbKeys = copyToHostAsBatch(distinctKeysTbl, partDataTypes) - (cbKeys, partitionIndexes) + val (keyHostCb, splitIds) = closeOnExcept(dataCb) { _ => + val (splitIds, distinctKeysTbl, keysCbTypes) = withResource(keysCb) { _ => + val keysCbTypes = GpuColumnVector.extractTypes(keysCb) + withResource(GpuColumnVector.from(keysCb)) { keysTable => + closeOnExcept(distinctAndSort(keysTable)) { distinctKeysTbl => + (splitIndexes(keysTable, distinctKeysTbl), distinctKeysTbl, keysCbTypes) + } } } + withResource(distinctKeysTbl) { _ => + (copyToHostAsBatch(distinctKeysTbl, keysCbTypes), splitIds) + } } - - val splits = closeOnExcept(cbKeys) { _ => - val spillableOutputColumnsBatch = - SpillableColumnarBatch(outputColumnsBatch, SpillPriorities.ACTIVE_ON_DECK_PRIORITY) - withRetryNoSplit(spillableOutputColumnsBatch) { spillable => - withResource(spillable.getColumnarBatch()) { outCb => + val splits = closeOnExcept(keyHostCb) { _ => + val scbOutput = closeOnExcept(dataCb)( _ => + SpillableColumnarBatch(dataCb, SpillPriorities.ACTIVE_ON_DECK_PRIORITY)) + withRetryNoSplit(scbOutput) { scb => + withResource(scb.getColumnarBatch()) { outCb => withResource(GpuColumnVector.from(outCb)) { outputColumnsTbl => withResource(outputColumnsTbl) { _ => - outputColumnsTbl.contiguousSplit(partitionIndexes: _*) + outputColumnsTbl.contiguousSplit(splitIds: _*) } } } } } - - val paths = closeOnExcept(splits) { _ => - withResource(cbKeys) { _ => - // Use the existing code to convert each row into a path. It would be nice to do this - // on the GPU, but the data should be small and there are things we cannot easily - // support on the GPU right now - import scala.collection.JavaConverters._ - // paths - cbKeys.rowIterator().asScala.map(getPartitionPath).toArray - } - } + // Build the split result withResource(splits) { _ => - // NOTE: the `zip` here has the effect that will remove an extra `ContiguousTable` - // added at the end of `splits` because we use `upperBound` to find the split points, - // and the last split point is the number of rows. - val outDataTypes = description.dataColumns.map(_.dataType).toArray - splits.zip(paths).zipWithIndex.map { case ((split, path), ix) => - splits(ix) = null - withResource(split) { _ => - SplitAndPath( - SpillableColumnarBatch( - split, outDataTypes, SpillPriorities.ACTIVE_BATCHING_PRIORITY), - path) - } + withResource(keyHostCb) { _ => + val getBucketId = genGetBucketIdFunc(keyHostCb) + val getNextPartPath = genGetPartitionPathFunc(keyHostCb) + val outDataTypes = description.dataColumns.map(_.dataType).toArray + (0 until keyHostCb.numRows()).safeMap { idx => + val split = splits(idx) + splits(idx) = null + closeOnExcept(split) { _ => + SplitPack( + SpillableColumnarBatch(split, outDataTypes, + SpillPriorities.ACTIVE_BATCHING_PRIORITY), + getNextPartPath(idx), getBucketId(idx)) + } + }.toArray } } } - private def getBatchToWrite( - partBatch: SpillableColumnarBatch, - savedStatus: Option[WriterStatusWithCaches]): SpillableColumnarBatch = { - val outDataTypes = description.dataColumns.map(_.dataType).toArray - if (savedStatus.isDefined && savedStatus.get.tableCaches.nonEmpty) { - // In the case where the concurrent partition writers fall back, we need to - // incorporate into the current part any pieces that are already cached - // in the `savedStatus`. Adding `partBatch` to what was saved could make a - // concatenated batch with number of rows larger than `maxRecordsPerFile`, - // so this concatenated result could be split later, which is not efficient. However, - // the concurrent writers are default off in Spark, so it is not clear if this - // code path is worth optimizing. - val concat: Table = - withResource(savedStatus.get.tableCaches) { subSpillableBatches => - val toConcat = subSpillableBatches :+ partBatch - - // clear the caches - savedStatus.get.tableCaches.clear() - - withRetryNoSplit(toConcat.toSeq) { spillables => - withResource(spillables.safeMap(_.getColumnarBatch())) { batches => - withResource(batches.map(GpuColumnVector.from)) { subTables => - Table.concatenate(subTables: _*) - } - } - } - } - withResource(concat) { _ => - SpillableColumnarBatch( - GpuColumnVector.from(concat, outDataTypes), - SpillPriorities.ACTIVE_ON_DECK_PRIORITY) - } - } else { - partBatch + /** + * Create a new writer according to the given writer id, and update the given + * writer status. It also closes the old writer in the writer status by default. + */ + protected final def renewOutWriter(newWriterId: WriterIndex, curWriterStatus: WriterAndStatus, + closeOldWriter: Boolean = true): Unit = { + if (closeOldWriter) { + releaseOutWriter(curWriterStatus) } + curWriterStatus.recordsInFile = 0 + curWriterStatus.writer = newWriter(newWriterId.partitionPath, newWriterId.bucketId, + curWriterStatus.fileCounter) + } + + /** + * Set up a writer to the given writer status for the given writer id. + * It will create a new one if needed. This is used when seeing a new partition + * and(or) a new bucket id. + */ + protected def setupCurrentWriter(newWriterId: WriterIndex, curWriterStatus: WriterAndStatus, + closeOldWriter: Boolean = true): Unit = { + renewOutWriter(newWriterId, curWriterStatus, closeOldWriter) } /** - * Write columnar batch. - * If the `cachesMap` is not empty, this single writer should restore the writers and caches in - * the `cachesMap`, this single writer should first combine the caches and current split data - * for a specific partition before write. + * Opens a new OutputWriter given a partition key and/or a bucket id. + * If bucket id is specified, we will append it to the end of the file name, but before the + * file extension, e.g. part-r-00009-ea518ad4-455a-4431-b471-d24e03814677-00002.gz.parquet * - * @param cb the column batch - * @param cachesMap used by `GpuDynamicPartitionDataConcurrentWriter` when fall back to single - * writer, single writer should handle the stored writers and the pending caches + * @param partDir the partition directory + * @param bucketId the bucket which all tuples being written by this OutputWriter belong to, + * currently does not support `bucketId`, it's always None + * @param fileCounter integer indicating the number of files to be written to `partDir` */ - protected def write( - batch: ColumnarBatch, - cachesMap: Option[mutable.HashMap[String, WriterStatusWithCaches]]): Unit = { - assert(isPartitioned) - assert(!isBucketed) + @scala.annotation.nowarn( + "msg=method newTaskTempFile.* in class FileCommitProtocol is deprecated" + ) + def newWriter(partDir: Option[String], bucketId: Option[Int], + fileCounter: Int): ColumnarOutputWriter = { + partDir.foreach(updatedPartitions.add) + // Currently will be empty + val bucketIdStr = bucketId.map(BucketingUtils.bucketIdToString).getOrElse("") - val maxRecordsPerFile = description.maxRecordsPerFile - val partDataTypes = description.partitionColumns.map(_.dataType).toArray - - // We have an entire batch that is sorted, so we need to split it up by key - // to get a batch per path - withResource(splitBatchByKeyAndClose(batch, partDataTypes)) { splitsAndPaths => - splitsAndPaths.zipWithIndex.foreach { case (SplitAndPath(partBatch, partPath), ix) => - // If we fall back from `GpuDynamicPartitionDataConcurrentWriter`, we should get the - // saved status - val savedStatus = updateCurrentWriterIfNeeded(partPath, cachesMap) - - // combine `partBatch` with any remnants for this partition for the concurrent - // writer fallback case in `savedStatus` - splitsAndPaths(ix) = null - val batchToWrite = getBatchToWrite(partBatch, savedStatus) - - // if the batch fits, write it as is, else split and write it. - if (!shouldSplitToFitMaxRecordsPerFile(maxRecordsPerFile, - currentWriterStatus.recordsInFile, batchToWrite.numRows())) { - writeUpdateMetricsAndClose(currentWriterStatus, batchToWrite) - } else { - // materialize an actual batch since we are going to split it - // on the GPU - val batchToSplit = withRetryNoSplit(batchToWrite) { _ => - batchToWrite.getColumnarBatch() - } - val maxRecordsPerFileSplits = splitToFitMaxRecordsAndClose( - batchToSplit, - maxRecordsPerFile, - currentWriterStatus.recordsInFile) - writeSplitBatchesAndClose(maxRecordsPerFileSplits, maxRecordsPerFile, partPath) - } - } + // This must be in a form that matches our bucketing format. See BucketingUtils. + val ext = f"$bucketIdStr.c$fileCounter%03d" + + description.outputWriterFactory.getFileExtension(taskAttemptContext) + + val customPath = partDir.flatMap { dir => + description.customPartitionLocations.get(PartitioningUtils.parsePathFragment(dir)) } + + val currentPath = if (customPath.isDefined) { + committer.newTaskTempFileAbsPath(taskAttemptContext, customPath.get, ext) + } else { + committer.newTaskTempFile(taskAttemptContext, partDir, ext) + } + + val outWriter = description.outputWriterFactory.newInstance( + path = currentPath, + dataSchema = description.dataColumns.toStructType, + context = taskAttemptContext) + + statsTrackers.foreach(_.newFile(currentPath)) + outWriter } - private def updateCurrentWriterIfNeeded( - partPath: String, - cachesMap: Option[mutable.HashMap[String, WriterStatusWithCaches]]): - Option[WriterStatusWithCaches] = { - var savedStatus: Option[WriterStatusWithCaches] = None - if (currentPartPath != partPath) { - val previousPartPath = currentPartPath - currentPartPath = partPath - - // see a new partition, close the old writer - val previousWriterStatus = currentWriterStatus - if (previousWriterStatus != null) { - releaseWriter(previousWriterStatus.outputWriter) - } + protected final def writeBatchPerMaxRecordsAndClose(scb: SpillableColumnarBatch, + writerId: WriterIndex, writerStatus: WriterAndStatus): Unit = { + val maxRecordsPerFile = description.maxRecordsPerFile + val recordsInFile = writerStatus.recordsInFile - if (cachesMap.isDefined) { - savedStatus = cachesMap.get.get(currentPartPath) - if (savedStatus.isDefined) { - // first try to restore the saved writer status, - // `GpuDynamicPartitionDataConcurrentWriter` may already opened the writer, and may - // have pending caches - currentWriterStatus = savedStatus.get.writerStatus - // entire batch that is sorted, see a new partition, the old write status is useless - cachesMap.get.remove(previousPartPath) - } else { - // create a new one - val writer = newWriter(partPath, None, 0) - currentWriterStatus = new WriterStatus(writer) - statsTrackers.foreach(_.newPartition()) + if (!shouldSplitToFitMaxRecordsPerFile(maxRecordsPerFile, recordsInFile, scb.numRows())) { + writeUpdateMetricsAndClose(scb, writerStatus) + } else { + val batch = withRetryNoSplit(scb) { scb => + scb.getColumnarBatch() + } + val splits = splitToFitMaxRecordsAndClose(batch, maxRecordsPerFile, recordsInFile) + withResource(splits) { _ => + val needNewWriterForFirstPart = recordsInFile >= maxRecordsPerFile + splits.zipWithIndex.foreach { case (part, partIx) => + if (partIx > 0 || needNewWriterForFirstPart) { + writerStatus.fileCounter += 1 + assert(writerStatus.fileCounter <= MAX_FILE_COUNTER, + s"File counter ${writerStatus.fileCounter} is beyond max value $MAX_FILE_COUNTER") + // will create a new file, so close the old writer + renewOutWriter(writerId, writerStatus) + } + splits(partIx) = null + writeUpdateMetricsAndClose(part, writerStatus) } - } else { - // create a new one - val writer = newWriter(partPath, None, 0) - currentWriterStatus = new WriterStatus(writer) - statsTrackers.foreach(_.newPartition()) } } - savedStatus } /** - * Write an array of spillable batches. + * Called just before updating the current writer status when seeing a new partition + * or a bucket. * - * Note: `spillableBatches` will be closed in this function. - * - * @param batches the SpillableColumnarBatch splits to be written - * @param maxRecordsPerFile the max number of rows per file - * @param partPath the partition directory + * @param curWriterId the current writer index */ - private def writeSplitBatchesAndClose( - spillableBatches: Array[SpillableColumnarBatch], - maxRecordsPerFile: Long, - partPath: String): Unit = { - var needNewWriter = currentWriterStatus.recordsInFile >= maxRecordsPerFile - withResource(spillableBatches) { _ => - spillableBatches.zipWithIndex.foreach { case (part, partIx) => - if (needNewWriter) { - currentWriterStatus.fileCounter += 1 - assert(currentWriterStatus.fileCounter <= MAX_FILE_COUNTER, - s"File counter ${currentWriterStatus.fileCounter} " + - s"is beyond max value $MAX_FILE_COUNTER") - - // will create a new file, close the old writer - if (currentWriterStatus != null) { - releaseWriter(currentWriterStatus.outputWriter) - } + protected def preUpdateCurrentWriterStatus(curWriterId: WriterIndex): Unit ={} - // create a new writer and update the writer in the status - currentWriterStatus.outputWriter = - newWriter(partPath, None, currentWriterStatus.fileCounter) - currentWriterStatus.recordsInFile = 0 + override def write(batch: ColumnarBatch): Unit = { + // The input batch that is entirely sorted, so split it up by partitions and (or) + // bucket ids, and write the split batches one by one. + withResource(splitBatchByKeyAndClose(batch)) { splitPacks => + splitPacks.zipWithIndex.foreach { case (SplitPack(sp, partPath, bucketId), i) => + val hasDiffPart = partPath != currentWriterId.partitionPath + val hasDiffBucket = bucketId != currentWriterId.bucketId + if (hasDiffPart || hasDiffBucket) { + preUpdateCurrentWriterStatus(currentWriterId) + if (hasDiffPart) { + currentWriterId.partitionPath = partPath + statsTrackers.foreach(_.newPartition()) + } + if (hasDiffBucket) { + currentWriterId.bucketId = bucketId + } + currentWriterStatus.fileCounter = 0 + setupCurrentWriter(currentWriterId, currentWriterStatus) } - spillableBatches(partIx) = null - writeUpdateMetricsAndClose(currentWriterStatus, part) - needNewWriter = true - } - } - } - - protected def writeUpdateMetricsAndClose( - writerStatus: WriterStatus, - spillableBatch: SpillableColumnarBatch): Unit = { - writerStatus.recordsInFile += - writerStatus.outputWriter.writeSpillableAndClose(spillableBatch, statsTrackers) - } - - /** Release all resources. */ - override def releaseResources(): Unit = { - // does not use `currentWriter`, single writer use `currentWriterStatus` - assert(currentWriter == null) - - if (currentWriterStatus != null) { - try { - currentWriterStatus.outputWriter.close() - statsTrackers.foreach(_.closeFile(currentWriterStatus.outputWriter.path())) - } finally { - currentWriterStatus = null + splitPacks(i) = null + writeBatchPerMaxRecordsAndClose(sp, currentWriterId, currentWriterStatus) } } } } /** - * Dynamic partition writer with concurrent writers, meaning multiple concurrent writers are opened - * for writing. + * Dynamic partition writer with concurrent writers, meaning multiple concurrent + * writers are opened for writing. * * The process has the following steps: * - Step 1: Maintain a map of output writers per each partition columns. Keep all @@ -754,18 +668,29 @@ class GpuDynamicPartitionDataConcurrentWriter( description: GpuWriteJobDescription, taskAttemptContext: TaskAttemptContext, committer: FileCommitProtocol, - spec: GpuConcurrentOutputWriterSpec, - taskContext: TaskContext) - extends GpuDynamicPartitionDataSingleWriter(description, taskAttemptContext, committer) { + spec: GpuConcurrentOutputWriterSpec) + extends GpuDynamicPartitionDataSingleWriter(description, taskAttemptContext, committer) + with Logging { - // Keep all the unclosed writers, key is partition directory string. - // Note: if fall back to sort-based mode, also use the opened writers in the map. - private val concurrentWriters = mutable.HashMap[String, WriterStatusWithCaches]() + /** Wrapper class for status and caches of a unique concurrent output writer. */ + private class WriterStatusWithBatches extends WriterAndStatus with AutoCloseable { + // caches for this partition or writer + val tableCaches: ListBuffer[SpillableColumnarBatch] = ListBuffer() - // guarantee to close the caches and writers when task is finished - onTaskCompletion(taskContext)(closeCachesAndWriters()) + // current device bytes for the above caches + var deviceBytes: Long = 0 - private val outDataTypes = description.dataColumns.map(_.dataType).toArray + override def close(): Unit = try { + releaseOutWriter(this) + } finally { + tableCaches.safeClose() + tableCaches.clear() + } + } + + // Keep all the unclosed writers, key is a partition path and(or) bucket id. + // Note: if fall back to sort-based mode, also use the opened writers in the map. + private val concurrentWriters = mutable.HashMap[WriterIndex, WriterStatusWithBatches]() private val partitionFlushSize = if (description.concurrentWriterPartitionFlushSize <= 0) { @@ -777,324 +702,196 @@ class GpuDynamicPartitionDataConcurrentWriter( description.concurrentWriterPartitionFlushSize } - // refer to current batch if should fall back to `single writer` - private var currentFallbackColumnarBatch: ColumnarBatch = _ + // Pending split batches that are not cached for the concurrent write because + // there are too many open writers, and it is going to fall back to the sorted + // sequential write. + private val pendingBatches: mutable.Queue[SpillableColumnarBatch] = mutable.Queue.empty - override def abort(): Unit = { - try { - closeCachesAndWriters() - } finally { - committer.abortTask(taskAttemptContext) + override def writeWithIterator(iterator: Iterator[ColumnarBatch]): Unit = { + // 1: try concurrent writer + while (iterator.hasNext && pendingBatches.isEmpty) { + // concurrent write and update the `concurrentWriters` map. + this.write(iterator.next()) } - } - /** - * State to indicate if we are falling back to sort-based writer. - * Because we first try to use concurrent writers, its initial value is false. - */ - private var fallBackToSortBased: Boolean = false + // 2: fall back to single write if the input is not all consumed. + if (pendingBatches.nonEmpty || iterator.hasNext) { + // sort the all the pending batches and ones in `iterator` + val pendingCbsIter = new Iterator[ColumnarBatch] { + override def hasNext: Boolean = pendingBatches.nonEmpty - private def writeWithSingleWriter(cb: ColumnarBatch): Unit = { - // invoke `GpuDynamicPartitionDataSingleWriter`.write, - // single writer will take care of the unclosed writers and the pending caches - // in `concurrentWriters` - super.write(cb, Some(concurrentWriters)) + override def next(): ColumnarBatch = { + if (!hasNext) { + throw new NoSuchElementException() + } + withResource(pendingBatches.dequeue())(_.getColumnarBatch()) + } + } + val sortIter = GpuOutOfCoreSortIterator(pendingCbsIter ++ iterator, + new GpuSorter(spec.sortOrder, spec.output), GpuSortExec.targetSize(spec.batchSize), + NoopMetric, NoopMetric, NoopMetric, NoopMetric) + while (sortIter.hasNext) { + // write with sort-based sequential writer + super.write(sortIter.next()) + } + } } - private def writeWithConcurrentWriter(cb: ColumnarBatch): Unit = { - this.write(cb) + /** This is for the fallback case, used to clean the writers map. */ + override def preUpdateCurrentWriterStatus(curWriterId: WriterIndex): Unit = { + concurrentWriters.remove(curWriterId) } - /** - * Write an iterator of column batch. - * - * @param iterator the iterator of column batch - */ - override def writeWithIterator(iterator: Iterator[ColumnarBatch]): Unit = { - // 1: try concurrent writer - while (iterator.hasNext && !fallBackToSortBased) { - // concurrently write and update the `concurrentWriters` map - // the `` will be updated - writeWithConcurrentWriter(iterator.next()) + /** This is for the fallback case, try to find the writer from cache first. */ + override def setupCurrentWriter(newWriterId: WriterIndex, writerStatus: WriterAndStatus, + closeOldWriter: Boolean): Unit = { + if (closeOldWriter) { + releaseOutWriter(writerStatus) } - - // 2: fall back to single writer - // Note single writer should restore writer status and handle the pending caches - if (fallBackToSortBased) { - // concat the put back batch and un-coming batches - val newIterator = Iterator.single(currentFallbackColumnarBatch) ++ iterator - // sort the all the batches in `iterator` - - val sortIterator: GpuOutOfCoreSortIterator = getSorted(newIterator) - while (sortIterator.hasNext) { - // write with sort-based single writer - writeWithSingleWriter(sortIterator.next()) - } + val oOpenStatus = concurrentWriters.get(newWriterId) + if (oOpenStatus.isDefined) { + val openStatus = oOpenStatus.get + writerStatus.writer = openStatus.writer + writerStatus.recordsInFile = openStatus.recordsInFile + writerStatus.fileCounter = openStatus.fileCounter + } else { + super.setupCurrentWriter(newWriterId, writerStatus, closeOldWriter = false) } } /** - * Sort the input iterator by out of core sort - * - * @param iterator the input iterator - * @return sorted iterator - */ - private def getSorted(iterator: Iterator[ColumnarBatch]): GpuOutOfCoreSortIterator = { - val gpuSortOrder: Seq[SortOrder] = spec.sortOrder - val output: Seq[Attribute] = spec.output - val sorter = new GpuSorter(gpuSortOrder, output) - - // use noop metrics below - val sortTime = NoopMetric - val opTime = NoopMetric - val outputBatch = NoopMetric - val outputRows = NoopMetric - - val targetSize = GpuSortExec.targetSize(spec.batchSize) - // out of core sort the entire iterator - GpuOutOfCoreSortIterator(iterator, sorter, targetSize, - opTime, sortTime, outputBatch, outputRows) - } - - /** - * concurrent write the columnar batch - * Note: if new partitions number in `cb` plus existing partitions number is greater than - * `maxWriters` limit, will put back the whole `cb` to 'single writer` + * The write path of concurrent writers * - * @param cb the columnar batch + * @param cb the columnar batch to be written */ override def write(cb: ColumnarBatch): Unit = { - assert(isPartitioned) - assert(!isBucketed) - if (cb.numRows() == 0) { // TODO https://github.com/NVIDIA/spark-rapids/issues/6453 // To solve above issue, I assume that an empty batch will be wrote for saving metadata. // If the assumption it's true, this concurrent writer should write the metadata here, // and should not run into below splitting and caching logic + cb.close() return } - // 1. combine partition columns and `cb` columns into a column array - val columnsWithPartition = ArrayBuffer[ColumnVector]() - - // this withResource is here to decrement the refcount of the partition columns - // that are projected out of `cb` - withResource(getPartitionColumnsAsBatch(cb)) { partitionColumnsBatch => - columnsWithPartition.appendAll(GpuColumnVector.extractBases(partitionColumnsBatch)) - } - - val cols = GpuColumnVector.extractBases(cb) - columnsWithPartition ++= cols - - // 2. group by the partition columns - // get sub-groups for each partition and get unique keys for each partition - val groupsAndKeys = withResource( - new Table(columnsWithPartition.toSeq: _*)) { colsWithPartitionTbl => - // [0, partition columns number - 1] - val partitionIndices = description.partitionColumns.indices - - // group by partition columns - val op = colsWithPartitionTbl.groupBy(partitionIndices: _*) - // return groups and uniq keys table - // Each row in uniq keys table is corresponding to a group - op.contiguousSplitGroupsAndGenUniqKeys() - } - - withResource(groupsAndKeys) { _ => - // groups number should equal to uniq keys number - assert(groupsAndKeys.getGroups.length == groupsAndKeys.getUniqKeyTable.getRowCount) - - val (groups, keys) = (groupsAndKeys.getGroups, groupsAndKeys.getUniqKeyTable) - - // 3. generate partition strings for all sub-groups in advance - val partDataTypes = description.partitionColumns.map(_.dataType).toArray - val dataTypes = GpuColumnVector.extractTypes(cb) - // generate partition string list for all groups - val partitionStrList = getPartitionStrList(keys, partDataTypes) - // key table is useless now - groupsAndKeys.closeUniqKeyTable() - - // 4. cache each group according to each partitionStr - withResource(groups) { _ => - - // first update fallBackToSortBased - withResource(cb) { _ => - var newPartitionNum = 0 - var groupIndex = 0 - while (!fallBackToSortBased && groupIndex < groups.length) { - // get the partition string - val partitionStr = partitionStrList(groupIndex) - groupIndex += 1 - if (!concurrentWriters.contains(partitionStr)) { - newPartitionNum += 1 - if (newPartitionNum + concurrentWriters.size >= spec.maxWriters) { - fallBackToSortBased = true - currentFallbackColumnarBatch = cb - // `cb` should be put back to single writer - GpuColumnVector.incRefCounts(cb) - } - } - } - } - - if (!fallBackToSortBased) { - // not fall, collect all caches - var groupIndex = 0 - while (groupIndex < groups.length) { - // get the partition string and group pair - val (partitionStr, group) = (partitionStrList(groupIndex), groups(groupIndex)) - val groupTable = group.getTable - groupIndex += 1 - - // create writer if encounter a new partition and put into `concurrentWriters` map - if (!concurrentWriters.contains(partitionStr)) { - val w = newWriter(partitionStr, None, 0) - val ws = new WriterStatus(w) - concurrentWriters.put(partitionStr, new WriterStatusWithCaches(ws)) - statsTrackers.foreach(_.newPartition()) - } - - // get data columns, tail part is data columns - val dataColumns = ArrayBuffer[ColumnVector]() - for (i <- description.partitionColumns.length until groupTable.getNumberOfColumns) { - dataColumns += groupTable.getColumn(i) - } - withResource(new Table(dataColumns.toSeq: _*)) { dataTable => - withResource(GpuColumnVector.from(dataTable, dataTypes)) { cb => - val outputCb = getOutputCb(cb) - // convert to spillable cache and add to the pending cache - val currWriterStatus = concurrentWriters(partitionStr) - // create SpillableColumnarBatch to take the owner of `outputCb` - currWriterStatus.tableCaches += SpillableColumnarBatch( - outputCb, SpillPriorities.ACTIVE_ON_DECK_PRIORITY) - currWriterStatus.deviceBytes += GpuColumnVector.getTotalDeviceMemoryUsed(outputCb) - } - } + // Split the batch and cache the result, along with opening the writers. + splitBatchToCacheAndClose(cb) + // Write the cached batches + val writeFunc: (WriterIndex, WriterStatusWithBatches) => Unit = + if (pendingBatches.nonEmpty) { + // Flush all the caches before going into sorted sequential write + writeOneCacheAndClose + } else { + // Still the concurrent write, so write out only partitions that size > threshold. + (wi, ws) => + if (ws.deviceBytes > partitionFlushSize) { + writeOneCacheAndClose(wi, ws) } - } } - } - - // 5. find all big enough partitions and write - if(!fallBackToSortBased) { - for ((partitionDir, ws) <- findBigPartitions(partitionFlushSize)) { - writeAndCloseCache(partitionDir, ws) - } - } - } - - private def getPartitionStrList( - uniqKeysTable: Table, partDataTypes: Array[DataType]): Array[String] = { - withResource(copyToHostAsBatch(uniqKeysTable, partDataTypes)) { oneRowCb => - import scala.collection.JavaConverters._ - oneRowCb.rowIterator().asScala.map(getPartitionPath).toArray + concurrentWriters.foreach { case (writerIdx, writerStatus) => + writeFunc(writerIdx, writerStatus) } } - private def writeAndCloseCache(partitionDir: String, status: WriterStatusWithCaches): Unit = { + private def writeOneCacheAndClose(writerId: WriterIndex, + status: WriterStatusWithBatches): Unit = { assert(status.tableCaches.nonEmpty) + // Concat tables if needed + val scbToWrite = GpuBatchUtils.concatSpillBatchesAndClose(status.tableCaches.toSeq).get + status.tableCaches.clear() + status.deviceBytes = 0 + writeBatchPerMaxRecordsAndClose(scbToWrite, writerId, status) + } - // get concat table or the single table - val spillableToWrite = if (status.tableCaches.length >= 2) { - // concat the sub batches to write in once. - val concatted = withRetryNoSplit(status.tableCaches.toSeq) { spillableSubBatches => - withResource(spillableSubBatches.safeMap(_.getColumnarBatch())) { subBatches => - withResource(subBatches.map(GpuColumnVector.from)) { subTables => - Table.concatenate(subTables: _*) - } - } + private def splitBatchToCacheAndClose(batch: ColumnarBatch): Unit = { + // Split batch to groups by sort columns, [partition and(or) bucket id column]. + val (keysAndGroups, keyTypes) = withResource(batch) { _ => + val (opBatch, keyTypes) = withResource(getKeysBatch(batch)) { keysBatch => + val combinedCb = GpuColumnVector.combineColumns(keysBatch, batch) + (combinedCb, GpuColumnVector.extractTypes(keysBatch)) } - withResource(concatted) { _ => - SpillableColumnarBatch( - GpuColumnVector.from(concatted, outDataTypes), - SpillPriorities.ACTIVE_ON_DECK_PRIORITY) + withResource(opBatch) { _ => + withResource(GpuColumnVector.from(opBatch)) { opTable => + (opTable.groupBy(keyTypes.indices: _*).contiguousSplitGroupsAndGenUniqKeys(), + keyTypes) + } } - } else { - // only one single table - status.tableCaches.head } - - status.tableCaches.clear() - - val maxRecordsPerFile = description.maxRecordsPerFile - if (!shouldSplitToFitMaxRecordsPerFile( - maxRecordsPerFile, status.writerStatus.recordsInFile, spillableToWrite.numRows())) { - writeUpdateMetricsAndClose(status.writerStatus, spillableToWrite) - } else { - val batchToSplit = withRetryNoSplit(spillableToWrite) { _ => - spillableToWrite.getColumnarBatch() - } - val splits = splitToFitMaxRecordsAndClose( - batchToSplit, - maxRecordsPerFile, - status.writerStatus.recordsInFile) - var needNewWriter = status.writerStatus.recordsInFile >= maxRecordsPerFile - withResource(splits) { _ => - splits.zipWithIndex.foreach { case (split, partIndex) => - if (needNewWriter) { - status.writerStatus.fileCounter += 1 - assert(status.writerStatus.fileCounter <= MAX_FILE_COUNTER, - s"File counter ${status.writerStatus.fileCounter} " + - s"is beyond max value $MAX_FILE_COUNTER") - status.writerStatus.outputWriter.close() - // start a new writer - val w = newWriter(partitionDir, None, status.writerStatus.fileCounter) - status.writerStatus.outputWriter = w - status.writerStatus.recordsInFile = 0L + // Copy keys table to host and make group batches spillable + val (keyHostCb, groups) = withResource(keysAndGroups) { _ => + // groups number should equal to uniq keys number + assert(keysAndGroups.getGroups.length == keysAndGroups.getUniqKeyTable.getRowCount) + closeOnExcept(copyToHostAsBatch(keysAndGroups.getUniqKeyTable, keyTypes)) { keyHostCb => + keysAndGroups.closeUniqKeyTable() + val allTypes = description.allColumns.map(_.dataType).toArray + val allColsIds = allTypes.indices.map(_ + keyTypes.length) + val gps = keysAndGroups.getGroups.safeMap { gp => + withResource(gp.getTable) { gpTable => + withResource(new Table(allColsIds.map(gpTable.getColumn): _*)) { allTable => + SpillableColumnarBatch(GpuColumnVector.from(allTable, allTypes), + SpillPriorities.ACTIVE_BATCHING_PRIORITY) + } } - splits(partIndex) = null - writeUpdateMetricsAndClose(status.writerStatus, split) - needNewWriter = true } + (keyHostCb, gps) } } - status.tableCaches.clear() - status.deviceBytes = 0 - } - - def closeCachesAndWriters(): Unit = { - // collect all caches and writers - val allResources = ArrayBuffer[AutoCloseable]() - allResources ++= concurrentWriters.values.flatMap(ws => ws.tableCaches) - allResources ++= concurrentWriters.values.map { ws => - new AutoCloseable() { - override def close(): Unit = { - ws.writerStatus.outputWriter.close() - statsTrackers.foreach(_.closeFile(ws.writerStatus.outputWriter.path())) + // Cache the result to either the map or the pending queue. + withResource(groups) { _ => + withResource(keyHostCb) { _ => + val getBucketId = genGetBucketIdFunc(keyHostCb) + val getNextPartPath = genGetPartitionPathFunc(keyHostCb) + var idx = 0 + while (idx < groups.length && concurrentWriters.size < spec.maxWriters) { + val writerId = new WriterIndex(getNextPartPath(idx), getBucketId(idx)) + val writerStatus = + concurrentWriters.getOrElseUpdate(writerId, new WriterStatusWithBatches) + if (writerStatus.writer == null) { + // a new partition or bucket, so create a writer + renewOutWriter(writerId, writerStatus, closeOldWriter = false) + } + withResource(groups(idx)) { gp => + groups(idx) = null + withResource(gp.getColumnarBatch()) { cb => + val dataScb = SpillableColumnarBatch(getDataColumnsAsBatch(cb), + SpillPriorities.ACTIVE_BATCHING_PRIORITY) + writerStatus.tableCaches.append(dataScb) + writerStatus.deviceBytes += dataScb.sizeInBytes + } + } + idx += 1 + } + if (idx < groups.length) { + // The open writers number reaches the limit, and still some partitions are + // not cached. Append to the queue for the coming fallback to the sorted + // sequential write. + groups.drop(idx).foreach(g => pendingBatches.enqueue(g)) + // Set to null to avoid double close + (idx until groups.length).foreach(groups(_) = null) + logInfo(s"Number of concurrent writers ${concurrentWriters.size} reaches " + + "the threshold. Fall back from concurrent writers to sort-based sequential" + + " writer.") } } } - - // safe close all the caches and writers - allResources.safeClose() - - // clear `concurrentWriters` map - concurrentWriters.values.foreach(ws => ws.tableCaches.clear()) - concurrentWriters.clear() } /** Release all resources. */ override def releaseResources(): Unit = { - // does not use `currentWriter`, only use the writers in the concurrent writer map - assert(currentWriter == null) - - if (fallBackToSortBased) { - // Note: we should close the last partition writer in the single writer. - super.releaseResources() - } + pendingBatches.safeClose() + pendingBatches.clear() // write all caches - concurrentWriters.filter(pair => pair._2.tableCaches.nonEmpty) - .foreach(pair => writeAndCloseCache(pair._1, pair._2)) + concurrentWriters.foreach { case (wi, ws) => + if (ws.tableCaches.nonEmpty) { + writeOneCacheAndClose(wi, ws) + } + } // close all resources - closeCachesAndWriters() - } - - private def findBigPartitions( - sizeThreshold: Long): mutable.Map[String, WriterStatusWithCaches] = { - concurrentWriters.filter(pair => pair._2.deviceBytes >= sizeThreshold) + concurrentWriters.values.toSeq.safeClose() + concurrentWriters.clear() + super.releaseResources() } } @@ -1105,7 +902,7 @@ class GpuDynamicPartitionDataConcurrentWriter( * @param bucketFileNamePrefix Prefix of output file name based on bucket id. */ case class GpuWriterBucketSpec( - bucketIdExpression: Expression, + bucketIdExpression: GpuExpression, bucketFileNamePrefix: Int => String) /** @@ -1134,4 +931,23 @@ class GpuWriteJobDescription( |Partition columns: ${partitionColumns.mkString(", ")} |Data columns: ${dataColumns.mkString(", ")} """.stripMargin) -} \ No newline at end of file +} + +object BucketIdMetaUtils { + // Tag for the bucketing write using Spark Murmur3Hash + def tagForBucketingWrite(meta: RapidsMeta[_, _, _], bucketSpec: Option[BucketSpec], + outputColumns: Seq[Attribute]): Unit = { + bucketSpec.foreach { bSpec => + // Create a Murmur3Hash expression to leverage the overriding types check. + val expr = Murmur3Hash( + bSpec.bucketColumnNames.map(n => outputColumns.find(_.name == n).get), + GpuHashPartitioningBase.DEFAULT_HASH_SEED) + val hashMeta = GpuOverrides.wrapExpr(expr, meta.conf, None) + hashMeta.tagForGpu() + if(!hashMeta.canThisBeReplaced) { + meta.willNotWorkOnGpu(s"Hashing for generating bucket IDs can not run" + + s" on GPU. Details: ${hashMeta.explain(all=false)}") + } + } + } +} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala index de066a5486d..d1a26dc80fc 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala @@ -41,7 +41,7 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.command.CreateDataSourceTableAsSelectCommand import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat -import org.apache.spark.sql.rapids.{GpuDataSourceBase, GpuOrcFileFormat} +import org.apache.spark.sql.rapids.{BucketIdMetaUtils, GpuDataSourceBase, GpuOrcFileFormat} import org.apache.spark.sql.rapids.shims.GpuCreateDataSourceTableAsSelectCommand @@ -56,9 +56,7 @@ final class CreateDataSourceTableAsSelectCommandMeta( private var gpuProvider: Option[ColumnarFileFormat] = None override def tagSelfForGpuInternal(): Unit = { - if (cmd.table.bucketSpec.isDefined) { - willNotWorkOnGpu("bucketing is not supported") - } + BucketIdMetaUtils.tagForBucketingWrite(this, cmd.table.bucketSpec, cmd.outputColumns) if (cmd.table.provider.isEmpty) { willNotWorkOnGpu("provider must be defined") } @@ -94,4 +92,4 @@ final class CreateDataSourceTableAsSelectCommandMeta( conf.stableSort, conf.concurrentWriterPartitionFlushSize) } -} \ No newline at end of file +} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala index 5e2601a0467..55d9bc53704 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala @@ -184,9 +184,8 @@ final class OptimizedCreateHiveTableAsSelectCommandMeta( willNotWorkOnGpu("partitioned writes are not supported") } - if (tableDesc.bucketSpec.isDefined) { - willNotWorkOnGpu("bucketing is not supported") - } + GpuBucketingUtils.tagForHiveBucketingWrite(this, tableDesc.bucketSpec, + cmd.outputColumns, false) val serde = tableDesc.storage.serde.getOrElse("").toLowerCase(Locale.ROOT) if (serde.contains("parquet")) { diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/spark311/GpuBucketingUtils.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/spark311/GpuBucketingUtils.scala new file mode 100644 index 00000000000..a604267d1d9 --- /dev/null +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/spark311/GpuBucketingUtils.scala @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/*** spark-rapids-shim-json-lines +{"spark": "311"} +{"spark": "312"} +{"spark": "313"} +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import com.nvidia.spark.rapids.RapidsMeta + +import org.apache.spark.sql.catalyst.catalog.BucketSpec +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.rapids.{BucketIdMetaUtils, GpuWriterBucketSpec} + +object GpuBucketingUtils { + + def getWriterBucketSpec( + bucketSpec: Option[BucketSpec], + dataColumns: Seq[Attribute], + options: Map[String, String], + forceHiveHash: Boolean): Option[GpuWriterBucketSpec] = { + bucketSpec.map { spec => + val bucketColumns = spec.bucketColumnNames.map(c => dataColumns.find(_.name == c).get) + if (forceHiveHash) { + // Forcely use HiveHash for Hive write commands for some customized Spark binaries. + // TODO: Cannot support this until we support Hive hash partitioning on the GPU + throw new UnsupportedOperationException("Hive hash partitioning is not supported" + + " on GPU") + } else { + // Spark bucketed table: use `HashPartitioning.partitionIdExpression` as bucket id + // expression, so that we can guarantee the data distribution is same between shuffle and + // bucketed data source, which enables us to only shuffle one side when join a bucketed + // table and a normal one. + val bucketIdExpression = GpuHashPartitioning(bucketColumns, spec.numBuckets) + .partitionIdExpression + GpuWriterBucketSpec(bucketIdExpression, (_: Int) => "") + } + } + } + + def isHiveHashBucketing(options: Map[String, String]): Boolean = false + + def getOptionsWithHiveBucketWrite(bucketSpec: Option[BucketSpec]): Map[String, String] = { + Map.empty + } + + def tagForHiveBucketingWrite(meta: RapidsMeta[_, _, _], bucketSpec: Option[BucketSpec], + outColumns: Seq[Attribute], forceHiveHash: Boolean): Unit = { + if (forceHiveHash) { + bucketSpec.foreach(_ => + meta.willNotWorkOnGpu("Hive Hashing for generating bucket IDs is not supported yet") + ) + } else { + BucketIdMetaUtils.tagForBucketingWrite(meta, bucketSpec, outColumns) + } + } +} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala index 034567d60e5..acdd53b74ab 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala @@ -36,7 +36,7 @@ spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.hive.rapids.shims import com.nvidia.spark.rapids.{DataFromReplacementRule, DataWritingCommandMeta, GpuDataWritingCommand, GpuOverrides, RapidsConf, RapidsMeta} -import com.nvidia.spark.rapids.shims.GpuCreateHiveTableAsSelectBase +import com.nvidia.spark.rapids.shims.{GpuBucketingUtils, GpuCreateHiveTableAsSelectBase} import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.catalyst.catalog.{CatalogTable, SessionCatalog} @@ -61,9 +61,8 @@ final class GpuCreateHiveTableAsSelectCommandMeta(cmd: CreateHiveTableAsSelectCo willNotWorkOnGpu("partitioned writes are not supported") } - if (tableDesc.bucketSpec.isDefined) { - willNotWorkOnGpu("bucketing is not supported") - } + GpuBucketingUtils.tagForHiveBucketingWrite(this, tableDesc.bucketSpec, + cmd.outputColumns, false) val catalog = spark.sessionState.catalog val tableExists = catalog.tableExists(tableDesc.identifier) @@ -137,4 +136,4 @@ case class GpuCreateHiveTableAsSelectCommand( // Do not support partitioned or bucketed writes override def requireSingleBatch: Boolean = false -} \ No newline at end of file +} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala index 2ea0301fa2c..3f59d6565a5 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala @@ -38,6 +38,7 @@ package org.apache.spark.sql.hive.rapids.shims import java.util.Locale import com.nvidia.spark.rapids.{ColumnarFileFormat, DataFromReplacementRule, DataWritingCommandMeta, GpuDataWritingCommand, RapidsConf, RapidsMeta} +import com.nvidia.spark.rapids.shims.GpuBucketingUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.hive.conf.HiveConf @@ -216,7 +217,9 @@ case class GpuInsertIntoHiveTable( hadoopConf = hadoopConf, fileFormat = fileFormat, outputLocation = tmpLocation.toString, - partitionAttributes = partitionAttributes) + partitionAttributes = partitionAttributes, + bucketSpec = table.bucketSpec, + options = GpuBucketingUtils.getOptionsWithHiveBucketWrite(table.bucketSpec)) if (partition.nonEmpty) { if (numDynamicPartitions > 0) { diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala index f788971a85f..4adbd7b2ef5 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala @@ -39,7 +39,7 @@ import java.util.{Date, UUID} import com.nvidia.spark.TimingUtils import com.nvidia.spark.rapids._ -import com.nvidia.spark.rapids.shims.RapidsFileSourceMetaUtils +import com.nvidia.spark.rapids.shims.{GpuBucketingUtils, RapidsFileSourceMetaUtils} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.mapreduce._ @@ -136,13 +136,8 @@ object GpuFileFormatWriter extends Logging { if (projectList.nonEmpty) GpuProjectExec(projectList, plan)() else plan } - val writerBucketSpec: Option[GpuWriterBucketSpec] = bucketSpec.map { spec => - // TODO: Cannot support this until we: - // support Hive hash partitioning on the GPU - throw new UnsupportedOperationException("GPU hash partitioning for bucketed data is not " - + "compatible with the CPU version") - } - + val writerBucketSpec = GpuBucketingUtils.getWriterBucketSpec(bucketSpec, dataColumns, + options, false) val sortColumns = bucketSpec.toSeq.flatMap { spec => spec.sortColumnNames.map(c => dataColumns.find(_.name == c).get) } @@ -328,8 +323,8 @@ object GpuFileFormatWriter extends Logging { } else { concurrentOutputWriterSpec match { case Some(spec) => - new GpuDynamicPartitionDataConcurrentWriter( - description, taskAttemptContext, committer, spec, TaskContext.get()) + new GpuDynamicPartitionDataConcurrentWriter(description, taskAttemptContext, + committer, spec) case _ => new GpuDynamicPartitionDataSingleWriter(description, taskAttemptContext, committer) } diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/GpuBucketingUtils.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/GpuBucketingUtils.scala new file mode 100644 index 00000000000..feb562fa9b8 --- /dev/null +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/GpuBucketingUtils.scala @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import com.nvidia.spark.rapids.RapidsMeta + +import org.apache.spark.sql.catalyst.catalog.BucketSpec +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.execution.datasources.BucketingUtils +import org.apache.spark.sql.rapids.GpuWriterBucketSpec + +object GpuBucketingUtils { + + def getWriterBucketSpec( + bucketSpec: Option[BucketSpec], + dataColumns: Seq[Attribute], + options: Map[String, String], + forceHiveHash: Boolean): Option[GpuWriterBucketSpec] = { + bucketSpec.map { spec => + val bucketColumns = spec.bucketColumnNames.map(c => dataColumns.find(_.name == c).get) + val shouldHiveCompatibleWrite = options.getOrElse( + BucketingUtils.optionForHiveCompatibleBucketWrite, "false").toBoolean + if (shouldHiveCompatibleWrite) { + // TODO: Cannot support this until we support Hive hash partitioning on the GPU + throw new UnsupportedOperationException("Hive hash partitioning is not supported" + + " on GPU") + } else { + // Spark bucketed table: use `HashPartitioning.partitionIdExpression` as bucket id + // expression, so that we can guarantee the data distribution is same between shuffle and + // bucketed data source, which enables us to only shuffle one side when join a bucketed + // table and a normal one. + val bucketIdExpression = GpuHashPartitioning(bucketColumns, spec.numBuckets) + .partitionIdExpression + GpuWriterBucketSpec(bucketIdExpression, (_: Int) => "") + } + } + } + + def isHiveHashBucketing(options: Map[String, String]): Boolean = { + options.getOrElse(BucketingUtils.optionForHiveCompatibleBucketWrite, "false").toBoolean + } + + def getOptionsWithHiveBucketWrite(bucketSpec: Option[BucketSpec]): Map[String, String] = { + bucketSpec + .map(_ => Map(BucketingUtils.optionForHiveCompatibleBucketWrite -> "true")) + .getOrElse(Map.empty) + } + + def tagForHiveBucketingWrite(meta: RapidsMeta[_, _, _], bucketSpec: Option[BucketSpec], + outColumns: Seq[Attribute], forceHiveHash: Boolean): Unit = { + bucketSpec.foreach(_ => + // From Spark330, Hive write always uses HiveHash to generate bucket IDs. + meta.willNotWorkOnGpu("Hive Hashing for generating bucket IDs is not supported yet") + ) + } +} diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala index faa550c0cb6..f51bd984bdc 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala @@ -30,10 +30,10 @@ package com.nvidia.spark.rapids.shims import com.nvidia.spark.rapids._ import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.execution.command.CreateDataSourceTableAsSelectCommand +import org.apache.spark.sql.execution.command.{CreateDataSourceTableAsSelectCommand, DataWritingCommand} import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat -import org.apache.spark.sql.rapids.{GpuDataSourceBase, GpuOrcFileFormat} +import org.apache.spark.sql.rapids.{BucketIdMetaUtils, GpuDataSourceBase, GpuOrcFileFormat} import org.apache.spark.sql.rapids.shims.GpuCreateDataSourceTableAsSelectCommand final class CreateDataSourceTableAsSelectCommandMeta( @@ -46,9 +46,9 @@ final class CreateDataSourceTableAsSelectCommandMeta( private var origProvider: Class[_] = _ override def tagSelfForGpu(): Unit = { - if (cmd.table.bucketSpec.isDefined) { - willNotWorkOnGpu("bucketing is not supported") - } + val outputColumns = + DataWritingCommand.logicalPlanOutputWithNames(cmd.query, cmd.outputColumnNames) + BucketIdMetaUtils.tagForBucketingWrite(this, cmd.table.bucketSpec, outputColumns) if (cmd.table.provider.isEmpty) { willNotWorkOnGpu("provider must be defined") } @@ -76,4 +76,4 @@ final class CreateDataSourceTableAsSelectCommandMeta( cmd.outputColumnNames, origProvider) } -} \ No newline at end of file +} diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala index 42fd5941025..b3103c3c76e 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala @@ -30,6 +30,7 @@ package org.apache.spark.sql.hive.rapids.shims import java.util.Locale import com.nvidia.spark.rapids.{ColumnarFileFormat, DataFromReplacementRule, DataWritingCommandMeta, GpuDataWritingCommand, RapidsConf, RapidsMeta} +import com.nvidia.spark.rapids.shims.GpuBucketingUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.hive.conf.HiveConf @@ -205,7 +206,9 @@ case class GpuInsertIntoHiveTable( hadoopConf = hadoopConf, fileFormat = fileFormat, outputLocation = tmpLocation.toString, - partitionAttributes = partitionAttributes) + partitionAttributes = partitionAttributes, + bucketSpec = table.bucketSpec, + options = GpuBucketingUtils.getOptionsWithHiveBucketWrite(table.bucketSpec)) if (partition.nonEmpty) { if (numDynamicPartitions > 0) { @@ -349,4 +352,4 @@ case class GpuInsertIntoHiveTable( } override def requireSingleBatch: Boolean = false // TODO: Re-evaluate. If partitioned or bucketed? -} \ No newline at end of file +} diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala index 53c17d2f946..e74bf979af9 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala @@ -197,9 +197,9 @@ final class OptimizedCreateHiveTableAsSelectCommandMeta( willNotWorkOnGpu("partitioned writes are not supported") } - if (tableDesc.bucketSpec.isDefined) { - willNotWorkOnGpu("bucketing is not supported") - } + val outputColumns = + DataWritingCommand.logicalPlanOutputWithNames(cmd.query, cmd.outputColumnNames) + GpuBucketingUtils.tagForHiveBucketingWrite(this, tableDesc.bucketSpec, outputColumns, false) val serde = tableDesc.storage.serde.getOrElse("").toLowerCase(Locale.ROOT) if (serde.contains("parquet")) { diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala index e7b3561f5fd..874d89353aa 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala @@ -31,7 +31,7 @@ import java.util.{Date, UUID} import com.nvidia.spark.TimingUtils import com.nvidia.spark.rapids._ -import com.nvidia.spark.rapids.shims.RapidsFileSourceMetaUtils +import com.nvidia.spark.rapids.shims.{GpuBucketingUtils, RapidsFileSourceMetaUtils} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.mapreduce._ @@ -119,13 +119,8 @@ object GpuFileFormatWriter extends Logging { .map(RapidsFileSourceMetaUtils.cleanupFileSourceMetadataInformation)) val dataColumns = finalOutputSpec.outputColumns.filterNot(partitionSet.contains) - val writerBucketSpec: Option[GpuWriterBucketSpec] = bucketSpec.map { spec => - // TODO: Cannot support this until we: - // support Hive hash partitioning on the GPU - throw new UnsupportedOperationException("GPU hash partitioning for bucketed data is not " - + "compatible with the CPU version") - } - + val writerBucketSpec = GpuBucketingUtils.getWriterBucketSpec(bucketSpec, dataColumns, + options, false) val sortColumns = bucketSpec.toSeq.flatMap { spec => spec.sortColumnNames.map(c => dataColumns.find(_.name == c).get) } @@ -419,8 +414,8 @@ object GpuFileFormatWriter extends Logging { } else { concurrentOutputWriterSpec match { case Some(spec) => - new GpuDynamicPartitionDataConcurrentWriter( - description, taskAttemptContext, committer, spec, TaskContext.get()) + new GpuDynamicPartitionDataConcurrentWriter(description, taskAttemptContext, + committer, spec) case _ => new GpuDynamicPartitionDataSingleWriter(description, taskAttemptContext, committer) } diff --git a/tests/src/test/scala/org/apache/spark/sql/rapids/GpuFileFormatDataWriterSuite.scala b/tests/src/test/scala/org/apache/spark/sql/rapids/GpuFileFormatDataWriterSuite.scala index 5aaeae2c7b9..d52c8b47ae7 100644 --- a/tests/src/test/scala/org/apache/spark/sql/rapids/GpuFileFormatDataWriterSuite.scala +++ b/tests/src/test/scala/org/apache/spark/sql/rapids/GpuFileFormatDataWriterSuite.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ package org.apache.spark.sql.rapids import ai.rapids.cudf.TableWriter -import com.nvidia.spark.rapids.{ColumnarOutputWriter, ColumnarOutputWriterFactory, GpuBoundReference, GpuColumnVector, RapidsBufferCatalog, RapidsDeviceMemoryStore, ScalableTaskCompletion} +import com.nvidia.spark.rapids.{ColumnarOutputWriter, ColumnarOutputWriterFactory, GpuColumnVector, GpuLiteral, RapidsBufferCatalog, RapidsDeviceMemoryStore, ScalableTaskCompletion} import com.nvidia.spark.rapids.Arm.{closeOnExcept, withResource} import com.nvidia.spark.rapids.jni.{GpuRetryOOM, GpuSplitAndRetryOOM} import org.apache.hadoop.conf.Configuration @@ -28,7 +28,6 @@ import org.scalatest.BeforeAndAfterEach import org.scalatest.funsuite.AnyFunSuite import org.scalatestplus.mockito.MockitoSugar.mock -import org.apache.spark.TaskContext import org.apache.spark.internal.io.FileCommitProtocol import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions.{Ascending, AttributeReference, ExprId, SortOrder} @@ -39,7 +38,6 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { private var mockJobDescription: GpuWriteJobDescription = _ - private var mockTaskContext: TaskContext = _ private var mockTaskAttemptContext: TaskAttemptContext = _ private var mockCommitter: FileCommitProtocol = _ private var mockOutputWriterFactory: ColumnarOutputWriterFactory = _ @@ -48,6 +46,7 @@ class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { private var allCols: Seq[AttributeReference] = _ private var partSpec: Seq[AttributeReference] = _ private var dataSpec: Seq[AttributeReference] = _ + private var bucketSpec: Option[GpuWriterBucketSpec] = None private var includeRetry: Boolean = false class NoTransformColumnarOutputWriter( @@ -102,9 +101,9 @@ class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { allCols = null partSpec = null dataSpec = null + bucketSpec = None mockJobDescription = mock[GpuWriteJobDescription] when(mockJobDescription.statsTrackers).thenReturn(Seq.empty) - mockTaskContext = mock[TaskContext] mockTaskAttemptContext = mock[TaskAttemptContext] mockCommitter = mock[FileCommitProtocol] mockOutputWriterFactory = mock[ColumnarOutputWriterFactory] @@ -130,8 +129,12 @@ class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { * It is used to setup certain mocks before `body` is executed. After execution, the * columns in the batches are checked for `refCount==0` (e.g. that they were closed). * @note it is assumed that the schema of each batch is identical. + * numBuckets > 0: Bucketing only + * numBuckets == 0: Partition only + * numBuckets < 0: Both partition and bucketing */ - def withColumnarBatchesVerifyClosed[V](cbs: Seq[ColumnarBatch])(body: => V): Unit = { + def withColumnarBatchesVerifyClosed[V]( + cbs: Seq[ColumnarBatch], numBuckets: Int = 0)(body: => V): Unit = { val allTypes = cbs.map(GpuColumnVector.extractTypes) allCols = Seq.empty dataSpec = Seq.empty @@ -140,8 +143,17 @@ class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { allCols = allTypes.head.zipWithIndex.map { case (dataType, colIx) => AttributeReference(s"col_$colIx", dataType, nullable = false)(ExprId(colIx)) } - partSpec = Seq(allCols.head) - dataSpec = allCols.tail + if (numBuckets <= 0) { + partSpec = Seq(allCols.head) + dataSpec = allCols.tail + } else { + dataSpec = allCols + } + if (numBuckets != 0) { + bucketSpec = Some(GpuWriterBucketSpec( + GpuPmod(GpuMurmur3Hash(Seq(allCols.last), 42), GpuLiteral(Math.abs(numBuckets))), + _ => "")) + } } val fields = new Array[StructField](allCols.size) allCols.zipWithIndex.foreach { case (col, ix) => @@ -153,6 +165,7 @@ class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { } when(mockJobDescription.dataColumns).thenReturn(dataSpec) when(mockJobDescription.partitionColumns).thenReturn(partSpec) + when(mockJobDescription.bucketSpec).thenReturn(bucketSpec) when(mockJobDescription.allColumns).thenReturn(allCols) try { body @@ -187,6 +200,20 @@ class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { new ColumnarBatch(cols, rowCount) } + def buildBatchWithPartitionedAndBucketCols( + partInts: Seq[Int], bucketInts: Seq[Int]): ColumnarBatch = { + assert(partInts.length == bucketInts.length) + val rowCount = partInts.size + val cols: Array[ColumnVector] = new Array[ColumnVector](3) + val partCol = ai.rapids.cudf.ColumnVector.fromInts(partInts: _*) + val dataCol = ai.rapids.cudf.ColumnVector.fromStrings(partInts.map(_.toString): _*) + val bucketCol = ai.rapids.cudf.ColumnVector.fromInts(bucketInts: _*) + cols(0) = GpuColumnVector.from(partCol, IntegerType) + cols(1) = GpuColumnVector.from(dataCol, StringType) + cols(2) = GpuColumnVector.from(bucketCol, IntegerType) + new ColumnarBatch(cols, rowCount) + } + def verifyClosed(cbs: Seq[ColumnarBatch]): Unit = { cbs.foreach { cb => val cols = GpuColumnVector.extractBases(cb) @@ -198,7 +225,6 @@ class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { def prepareDynamicPartitionSingleWriter(): GpuDynamicPartitionDataSingleWriter = { - when(mockJobDescription.bucketSpec).thenReturn(None) when(mockJobDescription.customPartitionLocations) .thenReturn(Map.empty[TablePartitionSpec, String]) @@ -212,13 +238,10 @@ class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { GpuDynamicPartitionDataConcurrentWriter = { val mockConfig = new Configuration() when(mockTaskAttemptContext.getConfiguration).thenReturn(mockConfig) - when(mockJobDescription.bucketSpec).thenReturn(None) when(mockJobDescription.customPartitionLocations) .thenReturn(Map.empty[TablePartitionSpec, String]) - // assume the first column is the partition-by column - val sortExpr = - GpuBoundReference(0, partSpec.head.dataType, nullable = false)(ExprId(0), "") - val sortSpec = Seq(SortOrder(sortExpr, Ascending)) + val sortSpec = (partSpec ++ bucketSpec.map(_.bucketIdExpression)) + .map(SortOrder(_, Ascending)) val concurrentSpec = GpuConcurrentOutputWriterSpec( maxWriters, allCols, batchSize, sortSpec) @@ -226,8 +249,7 @@ class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { mockJobDescription, mockTaskAttemptContext, mockCommitter, - concurrentSpec, - mockTaskContext)) + concurrentSpec)) } test("empty directory data writer") { @@ -317,18 +339,6 @@ class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { } } - test("dynamic partition data writer doesn't support bucketing") { - resetMocksWithAndWithoutRetry { - withColumnarBatchesVerifyClosed(Seq.empty) { - when(mockJobDescription.bucketSpec).thenReturn(Some(GpuWriterBucketSpec(null, null))) - assertThrows[UnsupportedOperationException] { - new GpuDynamicPartitionDataSingleWriter( - mockJobDescription, mockTaskAttemptContext, mockCommitter) - } - } - } - } - test("dynamic partition data writer without splits") { resetMocksWithAndWithoutRetry { // 4 partitions @@ -353,6 +363,35 @@ class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { } } + test("dynamic partition data writer bucketing write without splits") { + Seq(5, -5).foreach { numBuckets => + val (numWrites, numNewWriters) = if (numBuckets > 0) { // Bucket only + (6, 6) // 3 buckets + 3 buckets + } else { // partition and bucket + (10, 10) // 5 pairs + 5 pairs + } + resetMocksWithAndWithoutRetry { + val cb = buildBatchWithPartitionedAndBucketCols( + IndexedSeq(1, 1, 2, 2, 3, 3, 4, 4), + IndexedSeq(1, 1, 1, 1, 2, 2, 2, 3)) + val cb2 = buildBatchWithPartitionedAndBucketCols( + IndexedSeq(1, 2, 3, 4, 5), + IndexedSeq(1, 1, 2, 2, 3)) + val cbs = Seq(spy(cb), spy(cb2)) + withColumnarBatchesVerifyClosed(cbs, numBuckets) { + // setting to 9 then the writer won't split as no group has more than 9 rows + when(mockJobDescription.maxRecordsPerFile).thenReturn(9) + val dynamicSingleWriter = prepareDynamicPartitionSingleWriter() + dynamicSingleWriter.writeWithIterator(cbs.iterator) + dynamicSingleWriter.commit() + verify(mockOutputWriter, times(numWrites)).writeSpillableAndClose(any(), any()) + verify(dynamicSingleWriter, times(numNewWriters)).newWriter(any(), any(), any()) + verify(mockOutputWriter, times(numNewWriters)).close() + } + } + } + } + test("dynamic partition data writer with splits") { resetMocksWithAndWithoutRetry { val cb = buildBatchWithPartitionedCol(1, 1, 2, 2, 3, 3, 4, 4) @@ -399,6 +438,38 @@ class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { } } + test("dynamic partition concurrent data writer bucketing write without splits") { + Seq(5, -5).foreach { numBuckets => + val (numWrites, numNewWriters) = if (numBuckets > 0) { // Bucket only + (3, 3) // 3 distinct buckets in total + } else { // partition and bucket + (6, 6) // 6 distinct pairs in total + } + resetMocksWithAndWithoutRetry { + val cb = buildBatchWithPartitionedAndBucketCols( + IndexedSeq(1, 1, 2, 2, 3, 3, 4, 4), + IndexedSeq(1, 1, 1, 1, 2, 2, 2, 3)) + val cb2 = buildBatchWithPartitionedAndBucketCols( + IndexedSeq(1, 2, 3, 4, 5), + IndexedSeq(1, 1, 2, 2, 3)) + val cbs = Seq(spy(cb), spy(cb2)) + withColumnarBatchesVerifyClosed(cbs, numBuckets) { + // setting to 9 then the writer won't split as no group has more than 9 rows + when(mockJobDescription.maxRecordsPerFile).thenReturn(9) + // I would like to not flush on the first iteration of the `write` method + when(mockJobDescription.concurrentWriterPartitionFlushSize).thenReturn(1000) + val dynamicConcurrentWriter = + prepareDynamicPartitionConcurrentWriter(maxWriters = 20, batchSize = 100) + dynamicConcurrentWriter.writeWithIterator(cbs.iterator) + dynamicConcurrentWriter.commit() + verify(mockOutputWriter, times(numWrites)).writeSpillableAndClose(any(), any()) + verify(dynamicConcurrentWriter, times(numNewWriters)).newWriter(any(), any(), any()) + verify(mockOutputWriter, times(numNewWriters)).close() + } + } + } + } + test("dynamic partition concurrent data writer with splits and flush") { resetMocksWithAndWithoutRetry { val cb = buildBatchWithPartitionedCol(1, 1, 2, 2, 3, 3, 4, 4) @@ -438,8 +509,9 @@ class GpuFileFormatDataWriterSuite extends AnyFunSuite with BeforeAndAfterEach { prepareDynamicPartitionConcurrentWriter(maxWriters = 1, batchSize = 1) dynamicConcurrentWriter.writeWithIterator(cbs.iterator) dynamicConcurrentWriter.commit() - // 5 batches written, one per partition (no splitting) - verify(mockOutputWriter, times(5)) + // 6 batches written, one per partition (no splitting) plus one written by + // the concurrent writer. + verify(mockOutputWriter, times(6)) .writeSpillableAndClose(any(), any()) verify(dynamicConcurrentWriter, times(5)).newWriter(any(), any(), any()) // 5 files written because this is the single writer mode From 18ec4b2530f68ad2703e661cfb5a06aaaa2b2dea Mon Sep 17 00:00:00 2001 From: YanxuanLiu <104543031+YanxuanLiu@users.noreply.github.com> Date: Tue, 25 Jun 2024 09:04:37 +0800 Subject: [PATCH 41/79] upgrade actions version (#11086) Signed-off-by: YanxuanLiu --- .github/workflows/blossom-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml index 4b8071303c1..447f3d5049b 100644 --- a/.github/workflows/blossom-ci.yml +++ b/.github/workflows/blossom-ci.yml @@ -90,7 +90,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ fromJson(needs.Authorization.outputs.args).repo }} ref: ${{ fromJson(needs.Authorization.outputs.args).ref }} @@ -98,7 +98,7 @@ jobs: # repo specific steps - name: Setup java - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: adopt java-version: 8 From 86a905aac1544fef0554bad188c150b8e9720f91 Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Mon, 24 Jun 2024 22:54:55 -0700 Subject: [PATCH 42/79] Fixed Failing tests in arithmetic_ops_tests for Spark 4.0.0 [databricks] (#11044) * Fixed arithmetic_ops_tests * Signing off Signed-off-by: Raza Jafri * Added a mechanism to add ansi mode per test * Reverted unnecessary change to spark_init_internal.py * Corrected the year in the licence * Only set ansi conf to false when ansi_mode_disabled is set * Addressed review comments * Fixed the method name * Update integration_tests/src/main/python/conftest.py This handles cases like `cache_test.py` which should run with the default conf for `spark.sql.ansi.enabled`. --------- Signed-off-by: Raza Jafri Co-authored-by: MithunR --- .../src/main/python/arithmetic_ops_test.py | 77 ++++++++++++++----- integration_tests/src/main/python/conftest.py | 10 +++ integration_tests/src/main/python/marks.py | 3 +- .../src/main/python/spark_session.py | 9 ++- 4 files changed, 76 insertions(+), 23 deletions(-) diff --git a/integration_tests/src/main/python/arithmetic_ops_test.py b/integration_tests/src/main/python/arithmetic_ops_test.py index b75872ed8b2..d7fd941b97b 100644 --- a/integration_tests/src/main/python/arithmetic_ops_test.py +++ b/integration_tests/src/main/python/arithmetic_ops_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ from asserts import assert_gpu_and_cpu_are_equal_collect, assert_gpu_and_cpu_error, assert_gpu_fallback_collect, assert_gpu_and_cpu_are_equal_sql from data_gen import * -from marks import ignore_order, incompat, approximate_float, allow_non_gpu, datagen_overrides +from marks import ignore_order, incompat, approximate_float, allow_non_gpu, datagen_overrides, disable_ansi_mode from pyspark.sql.types import * from pyspark.sql.types import IntegralType from spark_session import * @@ -25,6 +25,10 @@ import pyspark.sql.utils from datetime import timedelta +_arithmetic_exception_string = 'java.lang.ArithmeticException' if is_before_spark_330() else \ + 'org.apache.spark.SparkArithmeticException' if is_before_spark_400() else \ + 'pyspark.errors.exceptions.captured.ArithmeticException' + # No overflow gens here because we just focus on verifying the fallback to CPU when # enabling ANSI mode. But overflows will fail the tests because CPU runs raise # exceptions. @@ -95,6 +99,7 @@ def _get_overflow_df(spark, data, data_type, expr): ).selectExpr(expr) @pytest.mark.parametrize('data_gen', _arith_data_gens, ids=idfn) +@disable_ansi_mode def test_addition(data_gen): data_type = data_gen.data_type assert_gpu_and_cpu_are_equal_collect( @@ -119,6 +124,7 @@ def test_addition_ansi_no_overflow(data_gen): conf=ansi_enabled_conf) @pytest.mark.parametrize('data_gen', _arith_data_gens, ids=idfn) +@disable_ansi_mode def test_subtraction(data_gen): data_type = data_gen.data_type assert_gpu_and_cpu_are_equal_collect( @@ -136,6 +142,7 @@ def test_subtraction(data_gen): DecimalGen(10, -2), DecimalGen(15, 3), DecimalGen(30, 12), DecimalGen(3, -3), DecimalGen(27, 7), DecimalGen(20, -3)], ids=idfn) @pytest.mark.parametrize('addOrSub', ['+', '-']) +@disable_ansi_mode def test_addition_subtraction_mixed(lhs, rhs, addOrSub): assert_gpu_and_cpu_are_equal_collect( lambda spark : two_col_df(spark, lhs, rhs).selectExpr(f"a {addOrSub} b") @@ -160,6 +167,7 @@ def test_subtraction_ansi_no_overflow(data_gen): _decimal_gen_38_10, _decimal_gen_38_neg10 ], ids=idfn) +@disable_ansi_mode def test_multiplication(data_gen): data_type = data_gen.data_type assert_gpu_and_cpu_are_equal_collect( @@ -203,6 +211,7 @@ def test_multiplication_ansi_overflow(): @pytest.mark.parametrize('rhs', [byte_gen, short_gen, int_gen, long_gen, DecimalGen(6, 3), DecimalGen(10, -2), DecimalGen(15, 3), DecimalGen(30, 12), DecimalGen(3, -3), DecimalGen(27, 7), DecimalGen(20, -3)], ids=idfn) +@disable_ansi_mode def test_multiplication_mixed(lhs, rhs): assert_gpu_and_cpu_are_equal_collect( lambda spark : two_col_df(spark, lhs, rhs).select( @@ -220,6 +229,7 @@ def test_float_multiplication_mixed(lhs, rhs): @pytest.mark.parametrize('data_gen', [double_gen, decimal_gen_32bit_neg_scale, DecimalGen(6, 3), DecimalGen(5, 5), DecimalGen(6, 0), DecimalGen(7, 4), DecimalGen(15, 0), DecimalGen(18, 0), DecimalGen(17, 2), DecimalGen(16, 4), DecimalGen(38, 21), DecimalGen(21, 17), DecimalGen(3, -2)], ids=idfn) +@disable_ansi_mode def test_division(data_gen): data_type = data_gen.data_type assert_gpu_and_cpu_are_equal_collect( @@ -232,6 +242,7 @@ def test_division(data_gen): @pytest.mark.parametrize('rhs', [byte_gen, short_gen, int_gen, long_gen, DecimalGen(4, 1), DecimalGen(5, 0), DecimalGen(5, 1), DecimalGen(10, 5)], ids=idfn) @pytest.mark.parametrize('lhs', [byte_gen, short_gen, int_gen, long_gen, DecimalGen(5, 3), DecimalGen(4, 2), DecimalGen(1, -2), DecimalGen(16, 1)], ids=idfn) +@disable_ansi_mode def test_division_mixed(lhs, rhs): assert_gpu_and_cpu_are_equal_collect( lambda spark : two_col_df(spark, lhs, rhs).select( @@ -242,12 +253,14 @@ def test_division_mixed(lhs, rhs): # instead of increasing the precision. So we have a second test that deals with a few of these use cases @pytest.mark.parametrize('rhs', [DecimalGen(30, 10), DecimalGen(28, 18)], ids=idfn) @pytest.mark.parametrize('lhs', [DecimalGen(27, 7), DecimalGen(20, -3)], ids=idfn) +@disable_ansi_mode def test_division_mixed_larger_dec(lhs, rhs): assert_gpu_and_cpu_are_equal_collect( lambda spark : two_col_df(spark, lhs, rhs).select( f.col('a'), f.col('b'), f.col('a') / f.col('b'))) +@disable_ansi_mode def test_special_decimal_division(): for precision in range(1, 39): for scale in range(-3, precision + 1): @@ -260,6 +273,7 @@ def test_special_decimal_division(): @approximate_float # we should get the perfectly correct answer for floats except when casting a decimal to a float in some corner cases. @pytest.mark.parametrize('rhs', [float_gen, double_gen], ids=idfn) @pytest.mark.parametrize('lhs', [DecimalGen(5, 3), DecimalGen(4, 2), DecimalGen(1, -2), DecimalGen(16, 1)], ids=idfn) +@disable_ansi_mode def test_float_division_mixed(lhs, rhs): assert_gpu_and_cpu_are_equal_collect( lambda spark : two_col_df(spark, lhs, rhs).select( @@ -269,6 +283,7 @@ def test_float_division_mixed(lhs, rhs): @pytest.mark.parametrize('data_gen', integral_gens + [ decimal_gen_32bit, decimal_gen_64bit, _decimal_gen_7_7, _decimal_gen_18_3, _decimal_gen_30_2, _decimal_gen_36_5, _decimal_gen_38_0], ids=idfn) +@disable_ansi_mode def test_int_division(data_gen): string_type = to_cast_string(data_gen.data_type) assert_gpu_and_cpu_are_equal_collect( @@ -282,12 +297,14 @@ def test_int_division(data_gen): @pytest.mark.parametrize('lhs', [DecimalGen(6, 5), DecimalGen(5, 4), DecimalGen(3, -2), _decimal_gen_30_2], ids=idfn) @pytest.mark.parametrize('rhs', [DecimalGen(13, 2), DecimalGen(6, 3), _decimal_gen_38_0, pytest.param(_decimal_gen_36_neg5, marks=pytest.mark.skipif(not is_before_spark_340() or is_databricks113_or_later(), reason='SPARK-41207'))], ids=idfn) +@disable_ansi_mode def test_int_division_mixed(lhs, rhs): assert_gpu_and_cpu_are_equal_collect( lambda spark : two_col_df(spark, lhs, rhs).selectExpr( 'a DIV b')) @pytest.mark.parametrize('data_gen', _arith_data_gens, ids=idfn) +@disable_ansi_mode def test_mod(data_gen): data_type = data_gen.data_type assert_gpu_and_cpu_are_equal_collect( @@ -308,6 +325,7 @@ def test_mod(data_gen): _decimal_gen_7_7] @pytest.mark.parametrize('data_gen', _pmod_gens, ids=idfn) +@disable_ansi_mode def test_pmod(data_gen): string_type = to_cast_string(data_gen.data_type) assert_gpu_and_cpu_are_equal_collect( @@ -321,6 +339,7 @@ def test_pmod(data_gen): @allow_non_gpu("ProjectExec", "Pmod") @pytest.mark.parametrize('data_gen', test_pmod_fallback_decimal_gens + [_decimal_gen_38_0, _decimal_gen_38_10], ids=idfn) +@disable_ansi_mode def test_pmod_fallback(data_gen): string_type = to_cast_string(data_gen.data_type) assert_gpu_fallback_collect( @@ -372,8 +391,10 @@ def test_cast_neg_to_decimal_err(): data_gen = _decimal_gen_7_7 if is_before_spark_322(): exception_content = "Decimal(compact,-120000000,20,0}) cannot be represented as Decimal(7, 7)" - elif is_databricks113_or_later() or not is_before_spark_340(): + elif is_databricks113_or_later() or not is_before_spark_340() and is_before_spark_400(): exception_content = "[NUMERIC_VALUE_OUT_OF_RANGE] -12 cannot be represented as Decimal(7, 7)" + elif not is_before_spark_400(): + exception_content = "[NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION] -12 cannot be represented as Decimal(7, 7)" else: exception_content = "Decimal(compact, -120000000, 20, 0) cannot be represented as Decimal(7, 7)" @@ -410,6 +431,7 @@ def test_mod_pmod_by_zero_not_ansi(data_gen): @pytest.mark.parametrize('rhs', [byte_gen, short_gen, int_gen, long_gen, DecimalGen(6, 3), DecimalGen(10, -2), DecimalGen(15, 3), DecimalGen(30, 12), DecimalGen(3, -3), DecimalGen(27, 7), DecimalGen(20, -3)], ids=idfn) +@disable_ansi_mode def test_mod_mixed(lhs, rhs): assert_gpu_and_cpu_are_equal_collect( lambda spark : two_col_df(spark, lhs, rhs).selectExpr(f"a % b")) @@ -417,6 +439,7 @@ def test_mod_mixed(lhs, rhs): # @pytest.mark.skipif(not is_databricks113_or_later() and not is_spark_340_or_later(), reason="https://github.com/NVIDIA/spark-rapids/issues/8330") @pytest.mark.parametrize('lhs', [DecimalGen(38,0), DecimalGen(37,2), DecimalGen(38,5), DecimalGen(38,-10), DecimalGen(38,7)], ids=idfn) @pytest.mark.parametrize('rhs', [DecimalGen(27,7), DecimalGen(30,10), DecimalGen(38,1), DecimalGen(36,0), DecimalGen(28,-7)], ids=idfn) +@disable_ansi_mode def test_mod_mixed_decimal128(lhs, rhs): assert_gpu_and_cpu_are_equal_collect( lambda spark : two_col_df(spark, lhs, rhs).selectExpr("a", "b", f"a % b")) @@ -424,6 +447,7 @@ def test_mod_mixed_decimal128(lhs, rhs): # Split into 4 tests to permute https://github.com/NVIDIA/spark-rapids/issues/7553 failures @pytest.mark.parametrize('lhs', [byte_gen, short_gen, int_gen, long_gen], ids=idfn) @pytest.mark.parametrize('rhs', [byte_gen, short_gen, int_gen, long_gen], ids=idfn) +@disable_ansi_mode def test_pmod_mixed_numeric(lhs, rhs): assert_gpu_and_cpu_are_equal_collect( lambda spark : two_col_df(spark, lhs, rhs).selectExpr(f"pmod(a, b)")) @@ -433,6 +457,7 @@ def test_pmod_mixed_numeric(lhs, rhs): DecimalGen(4, 2), DecimalGen(3, -2), DecimalGen(16, 7), DecimalGen(19, 0), DecimalGen(30, 10) ], ids=idfn) @pytest.mark.parametrize('rhs', [byte_gen, short_gen, int_gen, long_gen], ids=idfn) +@disable_ansi_mode def test_pmod_mixed_decimal_lhs(lhs, rhs): assert_gpu_fallback_collect( lambda spark : two_col_df(spark, lhs, rhs).selectExpr(f"pmod(a, b)"), @@ -443,6 +468,7 @@ def test_pmod_mixed_decimal_lhs(lhs, rhs): @pytest.mark.parametrize('rhs', [DecimalGen(6, 3), DecimalGen(10, -2), DecimalGen(15, 3), DecimalGen(30, 12), DecimalGen(3, -3), DecimalGen(27, 7), DecimalGen(20, -3) ], ids=idfn) +@disable_ansi_mode def test_pmod_mixed_decimal_rhs(lhs, rhs): assert_gpu_fallback_collect( lambda spark : two_col_df(spark, lhs, rhs).selectExpr(f"pmod(a, b)"), @@ -455,6 +481,7 @@ def test_pmod_mixed_decimal_rhs(lhs, rhs): @pytest.mark.parametrize('rhs', [DecimalGen(6, 3), DecimalGen(10, -2), DecimalGen(15, 3), DecimalGen(30, 12), DecimalGen(3, -3), DecimalGen(27, 7), DecimalGen(20, -3) ], ids=idfn) +@disable_ansi_mode def test_pmod_mixed_decimal(lhs, rhs): assert_gpu_fallback_collect( lambda spark : two_col_df(spark, lhs, rhs).selectExpr(f"pmod(a, b)"), @@ -466,6 +493,7 @@ def test_signum(data_gen): lambda spark : unary_op_df(spark, data_gen).selectExpr('signum(a)')) @pytest.mark.parametrize('data_gen', numeric_gens + _arith_decimal_gens_low_precision, ids=idfn) +@disable_ansi_mode def test_unary_minus(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark : unary_op_df(spark, data_gen).selectExpr('-a')) @@ -504,8 +532,7 @@ def test_unary_minus_ansi_overflow(data_type, value): assert_gpu_and_cpu_error( df_fun=lambda spark: _get_overflow_df(spark, [value], data_type, '-a').collect(), conf=ansi_enabled_conf, - error_message='java.lang.ArithmeticException' if is_before_spark_330() else \ - 'org.apache.spark.SparkArithmeticException') + error_message=_arithmetic_exception_string) # This just ends up being a pass through. There is no good way to force # a unary positive into a plan, because it gets optimized out, but this @@ -516,6 +543,7 @@ def test_unary_positive(data_gen): lambda spark : unary_op_df(spark, data_gen).selectExpr('+a')) @pytest.mark.parametrize('data_gen', numeric_gens + _arith_decimal_gens_low_precision, ids=idfn) +@disable_ansi_mode def test_abs(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark : unary_op_df(spark, data_gen).selectExpr('abs(a)')) @@ -556,10 +584,9 @@ def test_abs_ansi_overflow(data_type, value): GPU: One or more rows overflow for abs operation. """ assert_gpu_and_cpu_error( - df_fun=lambda spark: _get_overflow_df(spark, [value], data_type, 'abs(a)').collect(), - conf=ansi_enabled_conf, - error_message='java.lang.ArithmeticException' if is_before_spark_330() else \ - 'org.apache.spark.SparkArithmeticException') + df_fun=lambda spark: _get_overflow_df(spark, [value], data_type, 'abs(a)').collect(), + conf=ansi_enabled_conf, + error_message=_arithmetic_exception_string) @approximate_float @pytest.mark.parametrize('data_gen', double_gens, ids=idfn) @@ -613,7 +640,8 @@ def test_ceil_scale_zero(data_gen): @pytest.mark.parametrize('data_gen', [_decimal_gen_36_neg5, _decimal_gen_38_neg10], ids=idfn) def test_floor_ceil_overflow(data_gen): exception_type = "java.lang.ArithmeticException" if is_before_spark_330() and not is_databricks104_or_later() \ - else "SparkArithmeticException" + else "SparkArithmeticException" if is_before_spark_400() else \ + "pyspark.errors.exceptions.captured.ArithmeticException: [NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION]" assert_gpu_and_cpu_error( lambda spark: unary_op_df(spark, data_gen).selectExpr('floor(a)').collect(), conf={}, @@ -678,6 +706,7 @@ def test_shift_right_unsigned(data_gen): @approximate_float @datagen_overrides(seed=0, reason="https://github.com/NVIDIA/spark-rapids/issues/9350") @pytest.mark.parametrize('data_gen', _arith_data_gens_for_round, ids=idfn) +@disable_ansi_mode def test_decimal_bround(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark: unary_op_df(spark, data_gen).selectExpr( @@ -692,6 +721,7 @@ def test_decimal_bround(data_gen): @approximate_float @datagen_overrides(seed=0, reason="https://github.com/NVIDIA/spark-rapids/issues/9847") @pytest.mark.parametrize('data_gen', _arith_data_gens_for_round, ids=idfn) +@disable_ansi_mode def test_decimal_round(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark: unary_op_df(spark, data_gen).selectExpr( @@ -726,6 +756,7 @@ def doit(spark): @incompat @approximate_float +@disable_ansi_mode def test_non_decimal_round_overflow(): gen = StructGen([('byte_c', byte_gen), ('short_c', short_gen), ('int_c', int_gen), ('long_c', long_gen), @@ -1057,7 +1088,8 @@ def _div_overflow_exception_when(expr, ansi_enabled, is_lit=False): ansi_conf = {'spark.sql.ansi.enabled': ansi_enabled} err_exp = 'java.lang.ArithmeticException' if is_before_spark_330() else \ 'org.apache.spark.SparkArithmeticException' \ - if not is_lit or not is_spark_340_or_later() else "pyspark.errors.exceptions.captured.ArithmeticException" + if (not is_lit or not is_spark_340_or_later()) and is_before_spark_400() else \ + "pyspark.errors.exceptions.captured.ArithmeticException" err_mess = ': Overflow in integral divide' \ if is_before_spark_340() and not is_databricks113_or_later() else \ ': [ARITHMETIC_OVERFLOW] Overflow in integral divide' @@ -1123,7 +1155,7 @@ def test_add_overflow_with_ansi_enabled(data, tp, expr): assert_gpu_and_cpu_error( lambda spark: _get_overflow_df(spark, data, tp, expr).collect(), conf=ansi_enabled_conf, - error_message='java.lang.ArithmeticException' if is_before_spark_330() else 'SparkArithmeticException') + error_message=_arithmetic_exception_string) elif isinstance(tp, DecimalType): assert_gpu_and_cpu_error( lambda spark: _get_overflow_df(spark, data, tp, expr).collect(), @@ -1152,7 +1184,8 @@ def test_subtraction_overflow_with_ansi_enabled(data, tp, expr): assert_gpu_and_cpu_error( lambda spark: _get_overflow_df(spark, data, tp, expr).collect(), conf=ansi_enabled_conf, - error_message='java.lang.ArithmeticException' if is_before_spark_330() else 'SparkArithmeticException') + error_message='java.lang.ArithmeticException' if is_before_spark_330() else 'SparkArithmeticException' \ + if is_before_spark_400() else "pyspark.errors.exceptions.captured.ArithmeticException:") elif isinstance(tp, DecimalType): assert_gpu_and_cpu_error( lambda spark: _get_overflow_df(spark, data, tp, expr).collect(), @@ -1183,7 +1216,7 @@ def test_unary_minus_ansi_overflow_day_time_interval(ansi_enabled): assert_gpu_and_cpu_error( df_fun=lambda spark: _get_overflow_df(spark, [timedelta(microseconds=LONG_MIN)], DayTimeIntervalType(), '-a').collect(), conf={'spark.sql.ansi.enabled': ansi_enabled}, - error_message='SparkArithmeticException') + error_message='SparkArithmeticException' if is_before_spark_400() else "ArithmeticException") @pytest.mark.skipif(is_before_spark_330(), reason='DayTimeInterval is not supported before Pyspark 3.3.0') @pytest.mark.parametrize('ansi_enabled', ['false', 'true']) @@ -1224,7 +1257,7 @@ def test_add_overflow_with_ansi_enabled_day_time_interval(ansi_enabled): StructType([StructField('a', DayTimeIntervalType()), StructField('b', DayTimeIntervalType())]) ).selectExpr('a + b').collect(), conf={'spark.sql.ansi.enabled': ansi_enabled}, - error_message='SparkArithmeticException') + error_message=_arithmetic_exception_string) @pytest.mark.skipif(is_before_spark_330(), reason='DayTimeInterval is not supported before Pyspark 3.3.0') @pytest.mark.parametrize('ansi_enabled', ['false', 'true']) @@ -1244,7 +1277,7 @@ def test_subtraction_overflow_with_ansi_enabled_day_time_interval(ansi_enabled): StructType([StructField('a', DayTimeIntervalType()), StructField('b', DayTimeIntervalType())]) ).selectExpr('a - b').collect(), conf={'spark.sql.ansi.enabled': ansi_enabled}, - error_message='SparkArithmeticException') + error_message='SparkArithmeticException' if is_before_spark_400() else "ArithmeticException") @pytest.mark.skipif(is_before_spark_330(), reason='DayTimeInterval is not supported before Pyspark 3.3.0') def test_unary_positive_day_time_interval(): @@ -1303,7 +1336,8 @@ def _get_overflow_df_2cols(spark, data_types, values, expr): def test_day_time_interval_division_overflow(data_type, value_pair): exception_message = "SparkArithmeticException: Overflow in integral divide." \ if is_before_spark_340() and not is_databricks113_or_later() else \ - "SparkArithmeticException: [ARITHMETIC_OVERFLOW] Overflow in integral divide." + "SparkArithmeticException: [ARITHMETIC_OVERFLOW] Overflow in integral divide." if is_before_spark_400() else \ + "ArithmeticException: [ARITHMETIC_OVERFLOW] Overflow in integral divide." assert_gpu_and_cpu_error( df_fun=lambda spark: _get_overflow_df_2cols(spark, [DayTimeIntervalType(), data_type], value_pair, 'a / b').collect(), conf={}, @@ -1338,7 +1372,8 @@ def test_day_time_interval_division_round_overflow(data_type, value_pair): def test_day_time_interval_divided_by_zero(data_type, value_pair): exception_message = "SparkArithmeticException: Division by zero." \ if is_before_spark_340() and not is_databricks113_or_later() else \ - "SparkArithmeticException: [INTERVAL_DIVIDED_BY_ZERO] Division by zero" + "SparkArithmeticException: [INTERVAL_DIVIDED_BY_ZERO] Division by zero" if is_before_spark_400() else \ + "ArithmeticException: [INTERVAL_DIVIDED_BY_ZERO] Division by zero" assert_gpu_and_cpu_error( df_fun=lambda spark: _get_overflow_df_2cols(spark, [DayTimeIntervalType(), data_type], value_pair, 'a / b').collect(), conf={}, @@ -1349,7 +1384,8 @@ def test_day_time_interval_divided_by_zero(data_type, value_pair): def test_day_time_interval_divided_by_zero_scalar(zero_literal): exception_message = "SparkArithmeticException: Division by zero." \ if is_before_spark_340() and not is_databricks113_or_later() else \ - "SparkArithmeticException: [INTERVAL_DIVIDED_BY_ZERO] Division by zero." + "SparkArithmeticException: [INTERVAL_DIVIDED_BY_ZERO] Division by zero." if is_before_spark_400() else \ + "ArithmeticException: [INTERVAL_DIVIDED_BY_ZERO] Division by zero" assert_gpu_and_cpu_error( df_fun=lambda spark: _get_overflow_df_1col(spark, DayTimeIntervalType(), [timedelta(seconds=1)], 'a / ' + zero_literal).collect(), conf={}, @@ -1369,7 +1405,8 @@ def test_day_time_interval_divided_by_zero_scalar(zero_literal): def test_day_time_interval_scalar_divided_by_zero(data_type, value): exception_message = "SparkArithmeticException: Division by zero." \ if is_before_spark_340() and not is_databricks113_or_later() else \ - "SparkArithmeticException: [INTERVAL_DIVIDED_BY_ZERO] Division by zero." + "SparkArithmeticException: [INTERVAL_DIVIDED_BY_ZERO] Division by zero." if is_before_spark_400() else \ + "ArithmeticException: [INTERVAL_DIVIDED_BY_ZERO] Division by zero" assert_gpu_and_cpu_error( df_fun=lambda spark: _get_overflow_df_1col(spark, data_type, [value], 'INTERVAL 1 SECOND / a').collect(), conf={}, diff --git a/integration_tests/src/main/python/conftest.py b/integration_tests/src/main/python/conftest.py index 1adeb6964fd..6af40b99768 100644 --- a/integration_tests/src/main/python/conftest.py +++ b/integration_tests/src/main/python/conftest.py @@ -54,6 +54,7 @@ def array_columns_to_sort_locally(): _allow_any_non_gpu = False _non_gpu_allowed = [] +_per_test_ansi_mode_enabled = None def is_allowing_any_non_gpu(): return _allow_any_non_gpu @@ -61,6 +62,11 @@ def is_allowing_any_non_gpu(): def get_non_gpu_allowed(): return _non_gpu_allowed + +def is_per_test_ansi_mode_enabled(): + return _per_test_ansi_mode_enabled + + def get_validate_execs_in_gpu_plan(): return _validate_execs_in_gpu_plan @@ -210,10 +216,14 @@ def pytest_runtest_setup(item): global _allow_any_non_gpu global _non_gpu_allowed + global _per_test_ansi_mode_enabled _non_gpu_allowed_databricks = [] _allow_any_non_gpu_databricks = False non_gpu_databricks = item.get_closest_marker('allow_non_gpu_databricks') non_gpu = item.get_closest_marker('allow_non_gpu') + _per_test_ansi_mode_enabled = None if item.get_closest_marker('disable_ansi_mode') is None \ + else not item.get_closest_marker('disable_ansi_mode') + if non_gpu_databricks: if is_databricks_runtime(): if non_gpu_databricks.kwargs and non_gpu_databricks.kwargs['any']: diff --git a/integration_tests/src/main/python/marks.py b/integration_tests/src/main/python/marks.py index 1f326a75505..9a0bde11113 100644 --- a/integration_tests/src/main/python/marks.py +++ b/integration_tests/src/main/python/marks.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ allow_non_gpu_databricks = pytest.mark.allow_non_gpu_databricks allow_non_gpu = pytest.mark.allow_non_gpu +disable_ansi_mode = pytest.mark.disable_ansi_mode validate_execs_in_gpu_plan = pytest.mark.validate_execs_in_gpu_plan approximate_float = pytest.mark.approximate_float ignore_order = pytest.mark.ignore_order diff --git a/integration_tests/src/main/python/spark_session.py b/integration_tests/src/main/python/spark_session.py index c55f1976497..26388617fff 100644 --- a/integration_tests/src/main/python/spark_session.py +++ b/integration_tests/src/main/python/spark_session.py @@ -16,7 +16,7 @@ import calendar, time from datetime import date, datetime from contextlib import contextmanager, ExitStack -from conftest import is_allowing_any_non_gpu, get_non_gpu_allowed, get_validate_execs_in_gpu_plan, is_databricks_runtime, is_at_least_precommit_run, get_inject_oom_conf +from conftest import is_allowing_any_non_gpu, get_non_gpu_allowed, get_validate_execs_in_gpu_plan, is_databricks_runtime, is_at_least_precommit_run, get_inject_oom_conf, is_per_test_ansi_mode_enabled from pyspark.sql import DataFrame from pyspark.sql.types import TimestampType, DateType, _acceptable_types from spark_init_internal import get_spark_i_know_what_i_am_doing, spark_version @@ -41,7 +41,6 @@ def _from_scala_map(scala_map): # Many of these are redundant with default settings for the configs but are set here explicitly # to ensure any cluster settings do not interfere with tests that assume the defaults. _default_conf = { - 'spark.ansi.enabled': 'false', 'spark.rapids.sql.castDecimalToFloat.enabled': 'false', 'spark.rapids.sql.castFloatToDecimal.enabled': 'false', 'spark.rapids.sql.castFloatToIntegralTypes.enabled': 'false', @@ -127,6 +126,9 @@ def with_spark_session(func, conf={}): """Run func that takes a spark session as input with the given configs set.""" reset_spark_session_conf() _add_job_description(conf) + # Only set the ansi conf if not set by the test explicitly by setting the value in the dict + if "spark.sql.ansi.enabled" not in conf and is_per_test_ansi_mode_enabled() is not None: + conf["spark.sql.ansi.enabled"] = is_per_test_ansi_mode_enabled() _set_all_confs(conf) ret = func(_spark) _check_for_proper_return_values(ret) @@ -205,6 +207,9 @@ def is_before_spark_350(): def is_before_spark_351(): return spark_version() < "3.5.1" +def is_before_spark_400(): + return spark_version() < "4.0.0" + def is_spark_320_or_later(): return spark_version() >= "3.2.0" From 7a8690f5e2e4e9009c121e08c6429e2496b4f01c Mon Sep 17 00:00:00 2001 From: "Hongbin Ma (Mahone)" Date: Tue, 25 Jun 2024 15:39:47 +0800 Subject: [PATCH 43/79] fix duplicate counted metrics like op time for GpuCoalesceBatches (#11062) * with call site print, not good because some test cases by design will dup Signed-off-by: Hongbin Ma (Mahone) * done Signed-off-by: Hongbin Ma (Mahone) * add file Signed-off-by: Hongbin Ma (Mahone) * fix comiple Signed-off-by: Hongbin Ma (Mahone) * address review comments Signed-off-by: Hongbin Ma (Mahone) --------- Signed-off-by: Hongbin Ma (Mahone) --- .../spark/rapids/GpuCoalesceBatches.scala | 6 +- .../com/nvidia/spark/rapids/GpuExec.scala | 30 ++++++-- .../nvidia/spark/rapids/NvtxWithMetrics.scala | 18 +++-- .../nvidia/spark/rapids/MetricsSuite.scala | 68 +++++++++++++++++++ 4 files changed, 109 insertions(+), 13 deletions(-) create mode 100644 tests/src/test/scala/com/nvidia/spark/rapids/MetricsSuite.scala diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCoalesceBatches.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCoalesceBatches.scala index e6dc216d7e6..1afc03b177b 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCoalesceBatches.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCoalesceBatches.scala @@ -462,7 +462,7 @@ abstract class AbstractGpuCoalesceIterator( // If we have reached the cuDF limit once, proactively filter batches // after that first limit is reached. GpuFilter.filterAndClose(cbFromIter, inputFilterTier.get, - NoopMetric, NoopMetric, opTime) + NoopMetric, NoopMetric, NoopMetric) } else { Iterator(cbFromIter) } @@ -499,7 +499,7 @@ abstract class AbstractGpuCoalesceIterator( var filteredBytes = 0L if (hasAnyToConcat) { val filteredDowIter = GpuFilter.filterAndClose(concatAllAndPutOnGPU(), - filterTier, NoopMetric, NoopMetric, opTime) + filterTier, NoopMetric, NoopMetric, NoopMetric) while (filteredDowIter.hasNext) { closeOnExcept(filteredDowIter.next()) { filteredDownCb => filteredNumRows += filteredDownCb.numRows() @@ -512,7 +512,7 @@ abstract class AbstractGpuCoalesceIterator( // filterAndClose takes ownership of CB so we should not close it on a failure // anymore... val filteredCbIter = GpuFilter.filterAndClose(cb.release, filterTier, - NoopMetric, NoopMetric, opTime) + NoopMetric, NoopMetric, NoopMetric) while (filteredCbIter.hasNext) { closeOnExcept(filteredCbIter.next()) { filteredCb => val filteredWouldBeRows = filteredNumRows + filteredCb.numRows() diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExec.scala index ec87dd62d6c..d83f20113b2 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExec.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExec.scala @@ -152,12 +152,34 @@ sealed abstract class GpuMetric extends Serializable { def +=(v: Long): Unit def add(v: Long): Unit + private var isTimerActive = false + + final def tryActivateTimer(): Boolean = { + if (!isTimerActive) { + isTimerActive = true + true + } else { + false + } + } + + final def deactivateTimer(duration: Long): Unit = { + if (isTimerActive) { + isTimerActive = false + add(duration) + } + } + final def ns[T](f: => T): T = { - val start = System.nanoTime() - try { + if (tryActivateTimer()) { + val start = System.nanoTime() + try { + f + } finally { + deactivateTimer(System.nanoTime() - start) + } + } else { f - } finally { - add(System.nanoTime() - start) } } } diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/NvtxWithMetrics.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/NvtxWithMetrics.scala index 92a11f56123..538f117e50f 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/NvtxWithMetrics.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/NvtxWithMetrics.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,26 +32,32 @@ object NvtxWithMetrics { * by the amount of time spent in the range */ class NvtxWithMetrics(name: String, color: NvtxColor, val metrics: GpuMetric*) - extends NvtxRange(name, color) { + extends NvtxRange(name, color) { + val needTracks = metrics.map(_.tryActivateTimer()) private val start = System.nanoTime() override def close(): Unit = { val time = System.nanoTime() - start - metrics.foreach { metric => - metric += time + metrics.toSeq.zip(needTracks).foreach { pair => + if (pair._2) { + pair._1.deactivateTimer(time) + } } super.close() } } class MetricRange(val metrics: GpuMetric*) extends AutoCloseable { + val needTracks = metrics.map(_.tryActivateTimer()) private val start = System.nanoTime() override def close(): Unit = { val time = System.nanoTime() - start - metrics.foreach { metric => - metric += time + metrics.toSeq.zip(needTracks).foreach { pair => + if (pair._2) { + pair._1.deactivateTimer(time) + } } } } diff --git a/tests/src/test/scala/com/nvidia/spark/rapids/MetricsSuite.scala b/tests/src/test/scala/com/nvidia/spark/rapids/MetricsSuite.scala new file mode 100644 index 00000000000..580c5a2ed55 --- /dev/null +++ b/tests/src/test/scala/com/nvidia/spark/rapids/MetricsSuite.scala @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids + +import ai.rapids.cudf.NvtxColor +import com.nvidia.spark.rapids.Arm.withResource +import org.scalatest.funsuite.AnyFunSuite + +class MetricsSuite extends AnyFunSuite { + + test("GpuMetric.ns: duplicate timing on the same metrics") { + val m1 = new LocalGpuMetric() + m1.ns( + m1.ns( + Thread.sleep(100) + ) + ) + // if the timing is duplicated, the value should be around 200,000,000 + assert(m1.value < 100000000 * 1.5) + assert(m1.value > 100000000 * 0.5) + } + + test("MetricRange: duplicate timing on the same metrics") { + val m1 = new LocalGpuMetric() + val m2 = new LocalGpuMetric() + withResource(new MetricRange(m1, m2)) { _ => + withResource(new MetricRange(m2, m1)) { _ => + Thread.sleep(100) + } + } + + // if the timing is duplicated, the value should be around 200,000,000 + assert(m1.value < 100000000 * 1.5) + assert(m1.value > 100000000 * 0.5) + assert(m2.value < 100000000 * 1.5) + assert(m2.value > 100000000 * 0.5) + } + + test("NvtxWithMetrics: duplicate timing on the same metrics") { + val m1 = new LocalGpuMetric() + val m2 = new LocalGpuMetric() + withResource(new NvtxWithMetrics("a", NvtxColor.BLUE, m1, m2)) { _ => + withResource(new NvtxWithMetrics("b", NvtxColor.BLUE, m2, m1)) { _ => + Thread.sleep(100) + } + } + + // if the timing is duplicated, the value should be around 200,000,000 + assert(m1.value < 100000000 * 1.5) + assert(m1.value > 100000000 * 0.5) + assert(m2.value < 100000000 * 1.5) + assert(m2.value > 100000000 * 0.5) + } +} From b3b5b5e259a84aabca84bd960ba21ab70f672a9e Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Tue, 25 Jun 2024 06:29:34 -0700 Subject: [PATCH 44/79] Add GpuBucketingUtils shim to Spark 4.0.0 (#11092) * Add GpuBucketingUtils shim to Spark 4.0.0 * Signing off Signed-off-by: Raza Jafri --------- Signed-off-by: Raza Jafri --- .../nvidia/spark/rapids/shims/spark330/GpuBucketingUtils.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/GpuBucketingUtils.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/GpuBucketingUtils.scala index feb562fa9b8..0f7c9b4fd62 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/GpuBucketingUtils.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/GpuBucketingUtils.scala @@ -31,6 +31,7 @@ {"spark": "343"} {"spark": "350"} {"spark": "351"} +{"spark": "400"} spark-rapids-shim-json-lines ***/ package com.nvidia.spark.rapids.shims From 6455396a2c06f3cd8dfa9bf7692eebc57902440e Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 25 Jun 2024 11:19:53 -0700 Subject: [PATCH 45/79] Improve the diagnostics for 'conv' fallback explain (#11076) * Improve the diagnostics for 'conv' fallback explain Signed-off-by: Jihoon Son * don't use nil Signed-off-by: Jihoon Son * the bases should not be an empty string in the error message when the user input is not Signed-off-by: Jihoon Son * more user-friendly message * Update sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala Co-authored-by: Gera Shegalov --------- Signed-off-by: Jihoon Son Co-authored-by: Gera Shegalov --- .../src/main/python/string_test.py | 16 +++++++++++++++ .../spark/sql/rapids/stringFunctions.scala | 20 ++++++++++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/integration_tests/src/main/python/string_test.py b/integration_tests/src/main/python/string_test.py index 5631f13f13d..6ca0e1a1967 100644 --- a/integration_tests/src/main/python/string_test.py +++ b/integration_tests/src/main/python/string_test.py @@ -820,6 +820,22 @@ def test_conv_dec_to_from_hex(from_base, to_base, pattern): conf={'spark.rapids.sql.expression.Conv': True} ) +@pytest.mark.parametrize('from_base,to_base,expected_err_msg_prefix', + [ + pytest.param(10, 15, '15 is not a supported target radix', id='to_base_unsupported'), + pytest.param(11, 16, '11 is not a supported source radix', id='from_base_unsupported'), + pytest.param(9, 17, 'both 9 and 17 are not a supported radix', id='both_base_unsupported') + ]) +def test_conv_unsupported_base(from_base, to_base, expected_err_msg_prefix): + def do_conv(spark): + gen = StringGen() + df = unary_op_df(spark, gen).select('a', f.conv(f.col('a'), from_base, to_base)) + explain_str = spark.sparkContext._jvm.com.nvidia.spark.rapids.ExplainPlan.explainPotentialGpuPlan(df._jdf, "ALL") + unsupported_base_str = f'{expected_err_msg_prefix}, only literal 10 or 16 are supported for source and target radixes' + assert unsupported_base_str in explain_str + + with_cpu_session(do_conv) + format_number_gens = integral_gens + [DecimalGen(precision=7, scale=7), DecimalGen(precision=18, scale=0), DecimalGen(precision=18, scale=3), DecimalGen(precision=36, scale=5), DecimalGen(precision=36, scale=-5), DecimalGen(precision=38, scale=10), diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala index dc2845e4461..a435988686d 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala @@ -2084,11 +2084,25 @@ class GpuConvMeta( override def tagExprForGpu(): Unit = { val fromBaseLit = GpuOverrides.extractLit(expr.fromBaseExpr) val toBaseLit = GpuOverrides.extractLit(expr.toBaseExpr) + val errorPostfix = "only literal 10 or 16 are supported for source and target radixes" (fromBaseLit, toBaseLit) match { - case (Some(Literal(fromBaseVal, IntegerType)), Some(Literal(toBaseVal, IntegerType))) - if Set(fromBaseVal, toBaseVal).subsetOf(Set(10, 16)) => () + case (Some(Literal(fromBaseVal, IntegerType)), Some(Literal(toBaseVal, IntegerType))) => + def isBaseSupported(base: Any): Boolean = base == 10 || base == 16 + if (!isBaseSupported(fromBaseVal) && !isBaseSupported(toBaseVal)) { + willNotWorkOnGpu(because = s"both ${fromBaseVal} and ${toBaseVal} are not " + + s"a supported radix, ${errorPostfix}") + } else if (!isBaseSupported(fromBaseVal)) { + willNotWorkOnGpu(because = s"${fromBaseVal} is not a supported source radix, " + + s"${errorPostfix}") + } else if (!isBaseSupported(toBaseVal)) { + willNotWorkOnGpu(because = s"${toBaseVal} is not a supported target radix, " + + s"${errorPostfix}") + } case _ => - willNotWorkOnGpu(because = "only literal 10 or 16 for from_base and to_base are supported") + // This will never happen in production as the function signature enforces + // integer types for the bases, but nice to have an edge case handling. + willNotWorkOnGpu(because = "either source radix or target radix is not an integer " + + "literal, " + errorPostfix) } } From 34e6bc88b583fcb6e100fa507faff088994fdda3 Mon Sep 17 00:00:00 2001 From: MithunR Date: Tue, 25 Jun 2024 17:09:27 -0700 Subject: [PATCH 46/79] Disable ANSI mode for window function tests [databricks] (#11073) * Disable ANSI mode for window function tests. Fixes #11019. Window function tests fail on Spark 4.0 because of #5114 (and #5120 broadly), because spark-rapids does not support SUM, COUNT, and certain other aggregations in ANSI mode. This commit disables ANSI mode tests for the failing window function tests. These may be revisited, once error/overflow checking is available for ANSI mode in spark-rapids. Signed-off-by: MithunR * Switch from @ansi_mode_disabled to @disable_ansi_mode. --------- Signed-off-by: MithunR --- .../src/main/python/window_function_test.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/integration_tests/src/main/python/window_function_test.py b/integration_tests/src/main/python/window_function_test.py index af8bbbb55b3..44bc2a07d57 100644 --- a/integration_tests/src/main/python/window_function_test.py +++ b/integration_tests/src/main/python/window_function_test.py @@ -165,6 +165,8 @@ def test_float_window_min_max_all_nans(data_gen): .withColumn("max_b", f.max('a').over(w)) ) + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order @pytest.mark.parametrize('data_gen', [decimal_gen_128bit], ids=idfn) def test_decimal128_count_window(data_gen): @@ -177,6 +179,8 @@ def test_decimal128_count_window(data_gen): ' rows between 2 preceding and 10 following) as count_c_asc ' 'from window_agg_table') + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order @pytest.mark.parametrize('data_gen', [decimal_gen_128bit], ids=idfn) def test_decimal128_count_window_no_part(data_gen): @@ -189,6 +193,8 @@ def test_decimal128_count_window_no_part(data_gen): ' rows between 2 preceding and 10 following) as count_b_asc ' 'from window_agg_table') + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order @pytest.mark.parametrize('data_gen', decimal_gens, ids=idfn) def test_decimal_sum_window(data_gen): @@ -201,6 +207,8 @@ def test_decimal_sum_window(data_gen): ' rows between 2 preceding and 10 following) as sum_c_asc ' 'from window_agg_table') + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order @pytest.mark.parametrize('data_gen', decimal_gens, ids=idfn) def test_decimal_sum_window_no_part(data_gen): @@ -214,6 +222,7 @@ def test_decimal_sum_window_no_part(data_gen): 'from window_agg_table') +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order @pytest.mark.parametrize('data_gen', decimal_gens, ids=idfn) def test_decimal_running_sum_window(data_gen): @@ -227,6 +236,8 @@ def test_decimal_running_sum_window(data_gen): 'from window_agg_table', conf = {'spark.rapids.sql.batchSizeBytes': '100'}) + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order @pytest.mark.parametrize('data_gen', decimal_gens, ids=idfn) def test_decimal_running_sum_window_no_part(data_gen): @@ -302,6 +313,7 @@ def test_window_aggs_for_ranges_numeric_long_overflow(data_gen): 'from window_agg_table') +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # In a distributed setup the order of the partitions returned might be different, so we must ignore the order # but small batch sizes can make sort very slow, so do the final order by locally @ignore_order(local=True) @@ -352,6 +364,7 @@ def test_window_aggs_for_range_numeric_date(data_gen, batch_size): conf = conf) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # In a distributed setup the order of the partitions returned might be different, so we must ignore the order # but small batch sizes can make sort very slow, so do the final order by locally @ignore_order(local=True) @@ -396,6 +409,7 @@ def test_window_aggs_for_rows(data_gen, batch_size): conf = conf) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order(local=True) @pytest.mark.parametrize('batch_size', ['1000', '1g'], ids=idfn) @pytest.mark.parametrize('data_gen', [ @@ -482,6 +496,8 @@ def test_window_batched_unbounded(b_gen, batch_size): validate_execs_in_gpu_plan = ['GpuCachedDoublePassWindowExec'], conf = conf) + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # This is for aggregations that work with a running window optimization. They don't need to be batched # specially, but it only works if all of the aggregations can support this. # the order returned should be consistent because the data ends up in a single task (no partitioning) @@ -520,6 +536,7 @@ def test_rows_based_running_window_unpartitioned(b_gen, batch_size): conf = conf) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @pytest.mark.parametrize('batch_size', ['1000', '1g'], ids=idfn) # Testing multiple batch sizes. @pytest.mark.parametrize('a_gen', integral_gens + [string_gen, date_gen, timestamp_gen], ids=meta_idfn('data:')) @allow_non_gpu(*non_utc_allow) @@ -694,6 +711,7 @@ def test_window_running_rank(data_gen): conf = conf) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # This is for aggregations that work with a running window optimization. They don't need to be batched # specially, but it only works if all of the aggregations can support this. # In a distributed setup the order of the partitions returned might be different, so we must ignore the order @@ -738,6 +756,8 @@ def test_rows_based_running_window_partitioned(b_gen, c_gen, batch_size): conf = conf) + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order(local=True) @pytest.mark.parametrize('batch_size', ['1000', '1g'], ids=idfn) # Test different batch sizes. @pytest.mark.parametrize('part_gen', [int_gen, long_gen], ids=idfn) # Partitioning is not really the focus of the test. @@ -805,6 +825,7 @@ def must_test_sum_aggregation(gen): conf=conf) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # Test that we can do a running window sum on floats and doubles and decimal. This becomes problematic because we do the agg in parallel # which means that the result can switch back and forth from Inf to not Inf depending on the order of aggregations. # We test this by limiting the range of the values in the sum to never hit Inf, and by using abs so we don't have @@ -836,6 +857,7 @@ def test_window_running_float_decimal_sum(batch_size): conf = conf) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @approximate_float @ignore_order(local=True) @pytest.mark.parametrize('batch_size', ['1000', '1g'], ids=idfn) # Test different batch sizes. @@ -879,6 +901,7 @@ def window(oby_column): conf=conf) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # In a distributed setup the order of the partitions returned might be different, so we must ignore the order # but small batch sizes can make sort very slow, so do the final order by locally @ignore_order(local=True) @@ -1000,6 +1023,7 @@ def test_window_aggs_for_rows_lead_lag_on_arrays(a_gen, b_gen, c_gen, d_gen): ''') +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # lead and lag don't currently work for string columns, so redo the tests, but just for strings # without lead and lag # In a distributed setup the order of the partitions returned might be different, so we must ignore the order @@ -1107,6 +1131,8 @@ def test_window_aggs_lag_ignore_nulls_fallback(a_gen, b_gen, c_gen, d_gen): FROM window_agg_table ''') + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # Test for RANGE queries, with timestamp order-by expressions. # In a distributed setup the order of the partitions returned might be different, so we must ignore the order # but small batch sizes can make sort very slow, so do the final order by locally @@ -1155,6 +1181,7 @@ def test_window_aggs_for_ranges_timestamps(data_gen): conf = {'spark.rapids.sql.castFloatToDecimal.enabled': True}) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # In a distributed setup the order of the partitions returned might be different, so we must ignore the order # but small batch sizes can make sort very slow, so do the final order by locally @ignore_order(local=True) @@ -1201,6 +1228,7 @@ def test_window_aggregations_for_decimal_and_float_ranges(data_gen): conf={}) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # In a distributed setup the order of the partitions returned might be different, so we must ignore the order # but small batch sizes can make sort very slow, so do the final order by locally @ignore_order(local=True) @@ -1306,6 +1334,7 @@ def test_window_aggs_for_rows_collect_list(): conf={'spark.rapids.sql.window.collectList.enabled': True}) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # SortExec does not support array type, so sort the result locally. @ignore_order(local=True) # This test is more directed at Databricks and their running window optimization instead of ours @@ -1347,6 +1376,8 @@ def test_running_window_function_exec_for_all_aggs(): ''', conf={'spark.rapids.sql.window.collectList.enabled': True}) + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # Test the Databricks WindowExec which combines a WindowExec with a ProjectExec and provides the output # fields that we need to handle with an extra GpuProjectExec and we need the input expressions to compute # a window function of another window function case @@ -1668,6 +1699,8 @@ def do_it(spark): assert_gpu_fallback_collect(do_it, 'WindowExec') + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order(local=True) # single-level structs (no nested structs) are now supported by the plugin @pytest.mark.parametrize('part_gen', [StructGen([["a", long_gen]])], ids=meta_idfn('partBy:')) @@ -1731,6 +1764,8 @@ def do_it(spark): assert_gpu_and_cpu_are_equal_collect(do_it) + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order def test_unbounded_to_unbounded_window(): # This is specifically to test a bug that caused overflow issues when calculating @@ -1784,6 +1819,7 @@ def test_window_first_last_nth_ignore_nulls(data_gen): 'FROM window_agg_table') +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @tz_sensitive_test @allow_non_gpu(*non_supported_tz_allow) @ignore_order(local=True) @@ -1825,6 +1861,7 @@ def test_to_date_with_window_functions(): ) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order(local=True) @approximate_float @pytest.mark.parametrize('batch_size', ['1000', '1g'], ids=idfn) @@ -1881,6 +1918,7 @@ def spark_bugs_in_decimal_sorting(): return v < "3.1.4" or v < "3.3.1" or v < "3.2.3" or v < "3.4.0" +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order(local=True) @approximate_float @pytest.mark.parametrize('batch_size', ['1g'], ids=idfn) @@ -1925,6 +1963,7 @@ def test_window_aggs_for_negative_rows_unpartitioned(data_gen, batch_size): conf=conf) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order(local=True) @pytest.mark.parametrize('batch_size', ['1000', '1g'], ids=idfn) @pytest.mark.parametrize('data_gen', [ @@ -1964,6 +2003,7 @@ def test_window_aggs_for_batched_finite_row_windows_partitioned(data_gen, batch_ conf=conf) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order(local=True) @pytest.mark.parametrize('batch_size', ['1000', '1g'], ids=idfn) @pytest.mark.parametrize('data_gen', [ @@ -2003,6 +2043,7 @@ def test_window_aggs_for_batched_finite_row_windows_unpartitioned(data_gen, batc conf=conf) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 @ignore_order(local=True) @pytest.mark.parametrize('data_gen', [_grpkey_int_with_nulls,], ids=idfn) def test_window_aggs_for_batched_finite_row_windows_fallback(data_gen): From 3cb54c4ae2d43c7ba391dd619cf9f4d32c960e89 Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Thu, 27 Jun 2024 10:49:32 +0800 Subject: [PATCH 47/79] Fix some test issues in Spark UT and keep RapidsTestSettings update-to-date (#10997) * wip Signed-off-by: Haoyang Li * fix json suite Signed-off-by: Haoyang Li * wip Signed-off-by: Haoyang Li * update Signed-off-by: Haoyang Li * Remove all utc config and clean up Signed-off-by: Haoyang Li * hardcode timezone to LA in ci Signed-off-by: Haoyang Li * remove concat Signed-off-by: Haoyang Li * remove spark timezone settings and only keep java timezone settings Signed-off-by: Haoyang Li * remove unintensional comment Signed-off-by: Haoyang Li * delete a comment Signed-off-by: Haoyang Li * set timezone to utc for two suites to avoid fallback Signed-off-by: Haoyang Li * style Signed-off-by: Haoyang Li * after all Signed-off-by: Haoyang Li * add cast string to timestamp back to exclude after upmerge Signed-off-by: Haoyang Li --------- Signed-off-by: Haoyang Li --- .../sql/rapids/suites/RapidsCastSuite.scala | 49 +++++++++++++++ .../suites/RapidsJsonExpressionsSuite.scala | 16 ++++- .../suites/RapidsJsonFunctionsSuite.scala | 18 +++++- .../utils/RapidsSQLTestsBaseTrait.scala | 6 +- .../sql/rapids/utils/RapidsTestSettings.scala | 63 +++++++++---------- .../sql/rapids/utils/RapidsTestsTrait.scala | 5 +- 6 files changed, 113 insertions(+), 44 deletions(-) diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsCastSuite.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsCastSuite.scala index f3fec27f7f6..9b032c39e7c 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsCastSuite.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsCastSuite.scala @@ -19,7 +19,12 @@ spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.suites +import java.sql.Timestamp +import java.time.{LocalDateTime, ZoneId} + import org.apache.spark.sql.catalyst.expressions.{Cast, CastBase, CastSuite, Expression, Literal} +import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._ +import org.apache.spark.sql.catalyst.util.DateTimeUtils.getZoneId import org.apache.spark.sql.rapids.utils.RapidsTestsTrait import org.apache.spark.sql.types._ @@ -36,4 +41,48 @@ class RapidsCastSuite extends CastSuite with RapidsTestsTrait { Cast(lit, targetType, timeZoneId) } } + + private val specialTs = Seq( + "0001-01-01T00:00:00", // the fist timestamp of Common Era + "1582-10-15T23:59:59", // the cutover date from Julian to Gregorian calendar + "1970-01-01T00:00:00", // the epoch timestamp + "9999-12-31T23:59:59" // the last supported timestamp according to SQL standard + ) + + val outstandingTimezonesIds: Seq[String] = Seq( + "UTC", + PST.getId, + CET.getId, + "Africa/Dakar", + LA.getId, + "Asia/Urumqi", + "Asia/Hong_Kong", + "Europe/Brussels") + val outstandingZoneIds: Seq[ZoneId] = outstandingTimezonesIds.map(getZoneId) + + testRapids("SPARK-35711: cast timestamp without time zone to timestamp with local time zone") { + outstandingZoneIds.foreach { zoneId => + println(s"zoneId: $zoneId") + withDefaultTimeZone(zoneId) { + specialTs.foreach { s => + val input = LocalDateTime.parse(s) + val expectedTs = Timestamp.valueOf(s.replace("T", " ")) + checkEvaluation(cast(input, TimestampType), expectedTs) + } + } + } + } + + testRapids("SPARK-35719: cast timestamp with local time zone to timestamp without timezone") { + outstandingZoneIds.foreach { zoneId => + println(s"zoneId: $zoneId") + withDefaultTimeZone(zoneId) { + specialTs.foreach { s => + val input = Timestamp.valueOf(s.replace("T", " ")) + val expectedTs = LocalDateTime.parse(s) + checkEvaluation(cast(input, TimestampNTZType), expectedTs) + } + } + } + } } diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsJsonExpressionsSuite.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsJsonExpressionsSuite.scala index eb5fdc535e8..1c390e2479d 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsJsonExpressionsSuite.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsJsonExpressionsSuite.scala @@ -19,8 +19,22 @@ spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.suites +import java.util.TimeZone + import org.apache.spark.sql.catalyst.expressions.JsonExpressionsSuite import org.apache.spark.sql.rapids.utils.{RapidsJsonConfTrait, RapidsTestsTrait} class RapidsJsonExpressionsSuite - extends JsonExpressionsSuite with RapidsTestsTrait with RapidsJsonConfTrait {} + extends JsonExpressionsSuite with RapidsTestsTrait with RapidsJsonConfTrait { + + override def beforeAll(): Unit = { + super.beforeAll() + // Set timezone to UTC to avoid fallback, so that tests run on GPU to detect bugs + TimeZone.setDefault(TimeZone.getTimeZone("UTC")) + } + + override def afterAll(): Unit = { + TimeZone.setDefault(originalTimeZone) + super.afterAll() + } +} diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsJsonFunctionsSuite.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsJsonFunctionsSuite.scala index ebddc498202..1c39477134d 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsJsonFunctionsSuite.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsJsonFunctionsSuite.scala @@ -19,8 +19,24 @@ spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.suites +import java.util.TimeZone + import org.apache.spark.sql.JsonFunctionsSuite import org.apache.spark.sql.rapids.utils.{RapidsJsonConfTrait, RapidsSQLTestsTrait} class RapidsJsonFunctionsSuite - extends JsonFunctionsSuite with RapidsSQLTestsTrait with RapidsJsonConfTrait {} + extends JsonFunctionsSuite with RapidsSQLTestsTrait with RapidsJsonConfTrait { + + val originalTimeZone = TimeZone.getDefault + + override def beforeAll(): Unit = { + super.beforeAll() + // Set timezone to UTC to avoid fallback, so that tests run on GPU to detect bugs + TimeZone.setDefault(TimeZone.getTimeZone("UTC")) + } + + override def afterAll(): Unit = { + TimeZone.setDefault(originalTimeZone) + super.afterAll() + } +} diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsSQLTestsBaseTrait.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsSQLTestsBaseTrait.scala index f8b9d21d169..533f05be68b 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsSQLTestsBaseTrait.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsSQLTestsBaseTrait.scala @@ -151,10 +151,10 @@ object RapidsSQLTestsBaseTrait extends Logging { } def nativeSparkConf(origin: SparkConf, warehouse: String): SparkConf = { - // Timezone is fixed to UTC to allow timestamps to work by default - TimeZone.setDefault(TimeZone.getTimeZone("UTC")) // Add Locale setting Locale.setDefault(Locale.US) + // Spark use "America/Los_Angeles" as default timezone in tests + TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")) val conf = origin .set("spark.rapids.sql.enabled", "true") @@ -163,8 +163,6 @@ object RapidsSQLTestsBaseTrait extends Logging { "org.apache.spark.sql.rapids.ExecutionPlanCaptureCallback") .set("spark.sql.warehouse.dir", warehouse) .set("spark.sql.cache.serializer", "com.nvidia.spark.ParquetCachedBatchSerializer") - // TODO: remove hard coded UTC https://github.com/NVIDIA/spark-rapids/issues/10874 - .set("spark.sql.session.timeZone", "UTC") .set("spark.rapids.sql.explain", "ALL") // uncomment below config to run `strict mode`, where fallback to CPU is treated as fail // .set("spark.rapids.sql.test.enabled", "true") diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala index 63649376829..4f2e114f515 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala @@ -27,14 +27,13 @@ import org.apache.spark.sql.rapids.suites.{RapidsCastSuite, RapidsDataFrameAggre class RapidsTestSettings extends BackendTestSettings { enableSuite[RapidsCastSuite] - .exclude("Process Infinity, -Infinity, NaN in case insensitive manner", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) - .exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) - .exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) - .exclude("SPARK-35112: Cast string to day-time interval", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) - .exclude("SPARK-35735: Take into account day-time interval fields in cast", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) - .exclude("casting to fixed-precision decimals", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) - .exclude("SPARK-32828: cast from a derived user-defined type to a base type", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) - .exclude("cast string to timestamp", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) + .exclude("SPARK-35711: cast timestamp without time zone to timestamp with local time zone", WONT_FIX_ISSUE("https://issues.apache.org/jira/browse/SPARK-40851")) + .exclude("SPARK-35719: cast timestamp with local time zone to timestamp without timezone", WONT_FIX_ISSUE("https://issues.apache.org/jira/browse/SPARK-40851")) + .exclude("SPARK-35112: Cast string to day-time interval", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10980")) + .exclude("SPARK-35735: Take into account day-time interval fields in cast", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10980")) + .exclude("casting to fixed-precision decimals", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10809")) + .exclude("SPARK-32828: cast from a derived user-defined type to a base type", WONT_FIX_ISSUE("User-defined types are not supported")) + .exclude("cast string to timestamp", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/blob/main/docs/compatibility.md#string-to-timestamp")) .exclude("cast string to date", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10771")) enableSuite[RapidsDataFrameAggregateSuite] .exclude("collect functions", ADJUST_UT("order of elements in the array is non-deterministic in collect")) @@ -44,38 +43,32 @@ class RapidsTestSettings extends BackendTestSettings { .exclude("SPARK-19471: AggregationIterator does not initialize the generated result projection before using it", WONT_FIX_ISSUE("Codegen related UT, not applicable for GPU")) .exclude("SPARK-24788: RelationalGroupedDataset.toString with unresolved exprs should not fail", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10801")) enableSuite[RapidsJsonExpressionsSuite] - .exclude("from_json - invalid data", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("from_json - input=empty array, schema=struct, output=single row with null", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("from_json - input=empty object, schema=struct, output=single row with null", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("SPARK-20549: from_json bad UTF-8", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("to_json - array", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("to_json - array with single empty row", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("to_json - empty array", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("SPARK-21513: to_json support map[string, struct] to json", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("SPARK-21513: to_json support map[struct, struct] to json", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("SPARK-21513: to_json support map[string, integer] to json", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("to_json - array with maps", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("to_json - array with single map", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) - .exclude("from_json missing fields", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10849")) + .exclude("from_json - invalid data", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10891")) + .exclude("from_json - input=empty array, schema=struct, output=single row with null", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10907")) + .exclude("from_json - input=empty object, schema=struct, output=single row with null", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10910")) + .exclude("SPARK-20549: from_json bad UTF-8", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10911")) + .exclude("to_json - array", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10915")) + .exclude("to_json - array with single empty row", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10923")) + .exclude("to_json - empty array", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10924")) + .exclude("SPARK-21513: to_json support map[string, struct] to json", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10916")) + .exclude("SPARK-21513: to_json support map[struct, struct] to json", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10918")) + .exclude("SPARK-21513: to_json support map[string, integer] to json", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10919")) + .exclude("to_json - array with maps", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10920")) + .exclude("to_json - array with single map", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10921")) + .exclude("from_json missing fields", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10922")) enableSuite[RapidsJsonFunctionsSuite] - .exclude("from_json invalid json", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10852")) - .exclude("to_json - array", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10852")) - .exclude("to_json - map", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10852")) - .exclude("to_json - array of primitive types", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10852")) - .exclude("SPARK-33134: return partial results only for root JSON objects", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10852")) + .exclude("from_json invalid json", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10891")) + .exclude("to_json - array", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10894")) + .exclude("to_json - map", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10895")) + .exclude("to_json - array of primitive types", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10896")) + .exclude("SPARK-33134: return partial results only for root JSON objects", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10901")) enableSuite[RapidsJsonSuite] - .exclude("Casting long as timestamp", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) - .exclude("Write timestamps correctly with timestampFormat option and timeZone option", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) - .exclude("exception mode for parsing date/timestamp string", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10773")) enableSuite[RapidsMathFunctionsSuite] enableSuite[RapidsRegexpExpressionsSuite] enableSuite[RapidsStringExpressionsSuite] - .exclude("string substring_index function", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10775")) - .exclude("SPARK-22498: Concat should not generate codes beyond 64KB", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10775")) - .exclude("SPARK-22549: ConcatWs should not generate codes beyond 64KB", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10775")) - .exclude("SPARK-22550: Elt should not generate codes beyond 64KB", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10775")) - .exclude("SPARK-22603: FormatString should not generate codes beyond 64KB", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10775")) - .exclude("ParseUrl", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10775")) + .exclude("string substring_index function", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/8750")) + .exclude("SPARK-22550: Elt should not generate codes beyond 64KB", WONT_FIX_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10775")) + .exclude("SPARK-22603: FormatString should not generate codes beyond 64KB", WONT_FIX_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10775")) enableSuite[RapidsStringFunctionsSuite] } // scalastyle:on line.size.limit diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestsTrait.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestsTrait.scala index 69bd4532c71..d3aa8516679 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestsTrait.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestsTrait.scala @@ -56,7 +56,8 @@ trait RapidsTestsTrait extends RapidsTestsCommonTrait { super.beforeAll() initializeSession() _spark.sparkContext.setLogLevel("WARN") - TimeZone.setDefault(TimeZone.getTimeZone("UTC")) + // Spark use "America/Los_Angeles" as default timezone in tests + TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")) } override def afterAll(): Unit = { @@ -102,8 +103,6 @@ trait RapidsTestsTrait extends RapidsTestsCommonTrait { .config("spark.sql.queryExecutionListeners", "org.apache.spark.sql.rapids.ExecutionPlanCaptureCallback") .config("spark.sql.warehouse.dir", warehouse) - // TODO: remove hard coded UTC https://github.com/NVIDIA/spark-rapids/issues/10874 - .config("spark.sql.session.timeZone","UTC") .config("spark.rapids.sql.explain", "ALL") .config("spark.rapids.sql.test.isFoldableNonLitAllowed", "true") // uncomment below config to run `strict mode`, where fallback to CPU is treated as fail From 9dafc54c7c7287906d6cdc6ac4ef102c9d9a7f7b Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Thu, 27 Jun 2024 13:33:12 +0800 Subject: [PATCH 48/79] exclude a case based on JDK version (#11083) Signed-off-by: Haoyang Li --- .../rapids/utils/BackendTestSettings.scala | 40 +++++++++++++++++-- .../sql/rapids/utils/RapidsTestSettings.scala | 2 +- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/BackendTestSettings.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/BackendTestSettings.scala index e1aec1ffebc..a57b7802c9d 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/BackendTestSettings.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/BackendTestSettings.scala @@ -86,6 +86,19 @@ abstract class BackendTestSettings { case class ADJUST_UT(reason: String) extends ExcludeReason case class WONT_FIX_ISSUE(reason: String) extends ExcludeReason + protected def getJavaMajorVersion(): Int = { + val version = System.getProperty("java.version") + // Allow these formats: + // 1.8.0_72-ea + // 9-ea + // 9 + // 11.0.1 + val versionRegex = """(1\.)?(\d+)([._].+)?""".r + version match { + case versionRegex(_, major, _) => major.toInt + case _ => throw new IllegalStateException(s"Cannot parse java version: $version") + } + } final protected class SuiteSettings { private[utils] val inclusion: util.List[IncludeBase] = new util.ArrayList() @@ -96,42 +109,54 @@ abstract class BackendTestSettings { inclusion.add(Include(testNames: _*)) this } - def exclude(testNames: String, reason: ExcludeReason): SuiteSettings = { - exclusion.add(Exclude(testNames)) - excludeReasons.add(reason) + + def exclude(testNames: String, reason: ExcludeReason, condition: Boolean = true): + SuiteSettings = { + if (condition) { + exclusion.add(Exclude(testNames)) + excludeReasons.add(reason) + } this } + def includeRapidsTest(testName: String*): SuiteSettings = { inclusion.add(IncludeRapidsTest(testName: _*)) this } + def excludeRapidsTest(testName: String, reason: ExcludeReason): SuiteSettings = { exclusion.add(ExcludeRapidsTest(testName)) excludeReasons.add(reason) this } + def includeByPrefix(prefixes: String*): SuiteSettings = { inclusion.add(IncludeByPrefix(prefixes: _*)) this } + def excludeByPrefix(prefixes: String, reason: ExcludeReason): SuiteSettings = { exclusion.add(ExcludeByPrefix(prefixes)) excludeReasons.add(reason) this } + def includeRapidsTestsByPrefix(prefixes: String*): SuiteSettings = { inclusion.add(IncludeRapidsTestByPrefix(prefixes: _*)) this } + def excludeRapidsTestsByPrefix(prefixes: String, reason: ExcludeReason): SuiteSettings = { exclusion.add(ExcludeRadpisTestByPrefix(prefixes)) excludeReasons.add(reason) this } + def includeAllRapidsTests(): SuiteSettings = { inclusion.add(IncludeByPrefix(RAPIDS_TEST)) this } + def excludeAllRapidsTests(reason: ExcludeReason): SuiteSettings = { exclusion.add(ExcludeByPrefix(RAPIDS_TEST)) excludeReasons.add(reason) @@ -142,25 +167,31 @@ abstract class BackendTestSettings { protected trait IncludeBase { def isIncluded(testName: String): Boolean } + protected trait ExcludeBase { def isExcluded(testName: String): Boolean } + private case class Include(testNames: String*) extends IncludeBase { val nameSet: Set[String] = Set(testNames: _*) override def isIncluded(testName: String): Boolean = nameSet.contains(testName) } + private case class Exclude(testNames: String*) extends ExcludeBase { val nameSet: Set[String] = Set(testNames: _*) override def isExcluded(testName: String): Boolean = nameSet.contains(testName) } + private case class IncludeRapidsTest(testNames: String*) extends IncludeBase { val nameSet: Set[String] = testNames.map(name => RAPIDS_TEST + name).toSet override def isIncluded(testName: String): Boolean = nameSet.contains(testName) } + private case class ExcludeRapidsTest(testNames: String*) extends ExcludeBase { val nameSet: Set[String] = testNames.map(name => RAPIDS_TEST + name).toSet override def isExcluded(testName: String): Boolean = nameSet.contains(testName) } + private case class IncludeByPrefix(prefixes: String*) extends IncludeBase { override def isIncluded(testName: String): Boolean = { if (prefixes.exists(prefix => testName.startsWith(prefix))) { @@ -169,6 +200,7 @@ abstract class BackendTestSettings { false } } + private case class ExcludeByPrefix(prefixes: String*) extends ExcludeBase { override def isExcluded(testName: String): Boolean = { if (prefixes.exists(prefix => testName.startsWith(prefix))) { @@ -177,6 +209,7 @@ abstract class BackendTestSettings { false } } + private case class IncludeRapidsTestByPrefix(prefixes: String*) extends IncludeBase { override def isIncluded(testName: String): Boolean = { if (prefixes.exists(prefix => testName.startsWith(RAPIDS_TEST + prefix))) { @@ -185,6 +218,7 @@ abstract class BackendTestSettings { false } } + private case class ExcludeRadpisTestByPrefix(prefixes: String*) extends ExcludeBase { override def isExcluded(testName: String): Boolean = { if (prefixes.exists(prefix => testName.startsWith(RAPIDS_TEST + prefix))) { diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala index 4f2e114f515..ba8dcbe6efe 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala @@ -41,7 +41,7 @@ class RapidsTestSettings extends BackendTestSettings { .exclude("collect functions should be able to cast to array type with no null values", ADJUST_UT("order of elements in the array is non-deterministic in collect")) .exclude("SPARK-17641: collect functions should not collect null values", ADJUST_UT("order of elements in the array is non-deterministic in collect")) .exclude("SPARK-19471: AggregationIterator does not initialize the generated result projection before using it", WONT_FIX_ISSUE("Codegen related UT, not applicable for GPU")) - .exclude("SPARK-24788: RelationalGroupedDataset.toString with unresolved exprs should not fail", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10801")) + .exclude("SPARK-24788: RelationalGroupedDataset.toString with unresolved exprs should not fail", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10801"), (getJavaMajorVersion() >= 17)) enableSuite[RapidsJsonExpressionsSuite] .exclude("from_json - invalid data", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10891")) .exclude("from_json - input=empty array, schema=struct, output=single row with null", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/10907")) From 3b6c5cd44cb906374e4e56437b36129a7a78e9a0 Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Thu, 27 Jun 2024 23:15:24 -0700 Subject: [PATCH 49/79] Replaced spark3xx-common references to spark-shared [databricks] (#11066) * Replaced spark3xx-common references to spark-shared * Signing off Signed-off-by: Raza Jafri * addressed review comments * addressed review comments * removed todo as per review comment * Moving dependency to the related module because it was causing an error while running code coverage * Addressed review comments * Regenerated 2.13 poms --------- Signed-off-by: Raza Jafri --- build/coverage-report | 6 ++-- delta-lake/delta-20x/pom.xml | 4 +++ delta-lake/delta-21x/pom.xml | 4 +++ delta-lake/delta-22x/pom.xml | 4 +++ delta-lake/delta-23x/pom.xml | 4 +++ delta-lake/delta-24x/pom.xml | 4 +++ delta-lake/delta-spark330db/pom.xml | 4 +++ delta-lake/delta-spark332db/pom.xml | 4 +++ delta-lake/delta-spark341db/pom.xml | 4 +++ dist/scripts/binary-dedupe.sh | 5 ---- docs/dev/shims.md | 30 +++++++++---------- jenkins/spark-premerge-build.sh | 6 ++-- scala2.13/delta-lake/delta-20x/pom.xml | 4 +++ scala2.13/delta-lake/delta-21x/pom.xml | 4 +++ scala2.13/delta-lake/delta-22x/pom.xml | 4 +++ scala2.13/delta-lake/delta-23x/pom.xml | 4 +++ scala2.13/delta-lake/delta-24x/pom.xml | 4 +++ scala2.13/delta-lake/delta-spark330db/pom.xml | 4 +++ scala2.13/delta-lake/delta-spark332db/pom.xml | 4 +++ scala2.13/delta-lake/delta-spark341db/pom.xml | 4 +++ scala2.13/sql-plugin/pom.xml | 4 --- sql-plugin/pom.xml | 4 --- 22 files changed, 85 insertions(+), 34 deletions(-) diff --git a/build/coverage-report b/build/coverage-report index 69c96c050dc..75c86e55258 100755 --- a/build/coverage-report +++ b/build/coverage-report @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2020-2022, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -33,11 +33,11 @@ SOURCE_WITH_ARGS="--sourcefiles "$(echo $SOURCE_DIRS | sed -e 's/:/ --sourcefile rm -rf "$TMP_CLASS" mkdir -p "$TMP_CLASS" pushd "$TMP_CLASS" -jar xf "$DIST_JAR" com org rapids spark3xx-common "spark${SPK_VER}/" +jar xf "$DIST_JAR" com org rapids spark-shared "spark${SPK_VER}/" # extract the .class files in udf jar and replace the existing ones in spark3xx-ommon and spark$SPK_VER # because the class files in udf jar will be modified in aggregator's shade phase jar xf "$UDF_JAR" com/nvidia/spark/udf -rm -rf com/nvidia/shaded/ org/openucx/ spark3xx-common/com/nvidia/spark/udf/ spark${SPK_VER}/com/nvidia/spark/udf/ +rm -rf com/nvidia/shaded/ org/openucx/ spark-shared/com/nvidia/spark/udf/ spark${SPK_VER}/com/nvidia/spark/udf/ popd if [ ! -f "$JDEST" ]; then diff --git a/delta-lake/delta-20x/pom.xml b/delta-lake/delta-20x/pom.xml index dfc34fad987..be646b807f5 100644 --- a/delta-lake/delta-20x/pom.xml +++ b/delta-lake/delta-20x/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/delta-lake/delta-21x/pom.xml b/delta-lake/delta-21x/pom.xml index 8770a7e3d9d..f2915278625 100644 --- a/delta-lake/delta-21x/pom.xml +++ b/delta-lake/delta-21x/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/delta-lake/delta-22x/pom.xml b/delta-lake/delta-22x/pom.xml index be9e122e5df..0be1371f4a2 100644 --- a/delta-lake/delta-22x/pom.xml +++ b/delta-lake/delta-22x/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/delta-lake/delta-23x/pom.xml b/delta-lake/delta-23x/pom.xml index 97207dca741..b4bd721dd55 100644 --- a/delta-lake/delta-23x/pom.xml +++ b/delta-lake/delta-23x/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/delta-lake/delta-24x/pom.xml b/delta-lake/delta-24x/pom.xml index 19e0be3c90d..5573709b446 100644 --- a/delta-lake/delta-24x/pom.xml +++ b/delta-lake/delta-24x/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/delta-lake/delta-spark330db/pom.xml b/delta-lake/delta-spark330db/pom.xml index abc57f793f8..3ee901eef2c 100644 --- a/delta-lake/delta-spark330db/pom.xml +++ b/delta-lake/delta-spark330db/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/delta-lake/delta-spark332db/pom.xml b/delta-lake/delta-spark332db/pom.xml index 4511a76be71..92964a59e17 100644 --- a/delta-lake/delta-spark332db/pom.xml +++ b/delta-lake/delta-spark332db/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/delta-lake/delta-spark341db/pom.xml b/delta-lake/delta-spark341db/pom.xml index 6532062af68..6eaa3fb8c32 100644 --- a/delta-lake/delta-spark341db/pom.xml +++ b/delta-lake/delta-spark341db/pom.xml @@ -38,6 +38,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/dist/scripts/binary-dedupe.sh b/dist/scripts/binary-dedupe.sh index 356b0b4dbae..970840f59e3 100755 --- a/dist/scripts/binary-dedupe.sh +++ b/dist/scripts/binary-dedupe.sh @@ -127,11 +127,6 @@ done mv "$SPARK_SHARED_DIR" parallel-world/ -# TODO further dedupe by FEATURE version lines: -# spark30x-common -# spark31x-common -# spark32x-common - # Verify that all class files in the conventional jar location are bitwise # identical regardless of the Spark-version-specific jar. # diff --git a/docs/dev/shims.md b/docs/dev/shims.md index b6290287314..218adbc99d4 100644 --- a/docs/dev/shims.md +++ b/docs/dev/shims.md @@ -69,7 +69,7 @@ Spark 3.0.2's URLs: ```text jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/ -jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark3xx-common/ +jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark-shared/ jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark302/ ``` @@ -77,7 +77,7 @@ Spark 3.2.0's URLs : ```text jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/ -jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark3xx-common/ +jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark-shared/ jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark320/ ``` @@ -143,7 +143,7 @@ This has two pre-requisites: 1. The .class file with the bytecode is bitwise-identical among the currently supported Spark versions. To verify this you can inspect the dist jar and check -if the class file is under `spark3xx-common` jar entry. If this is not the case then +if the class file is under `spark-shared` jar entry. If this is not the case then code should be refactored until all discrepancies are shimmed away. 1. The transitive closure of the classes compile-time-referenced by `A` should have the property above. @@ -181,28 +181,28 @@ mv org com ai public/ and you will see the dependencies of `public` classes. By design `public` classes should have only edges only to other `public` classes in the dist jar. -Execute `jdeps` against `public`, `spark3xx-common` and an *exactly one* parallel +Execute `jdeps` against `public`, `spark-shared` and an *exactly one* parallel world such as `spark330` ```bash ${JAVA_HOME}/bin/jdeps -v \ -dotoutput /tmp/jdeps330 \ -regex '(com|org)\..*\.rapids\..*' \ - public spark3xx-common spark330 + public spark-shared spark330 ``` This will produce three DOT files for each "archive" with directed edges for a class in the archive to a class either in this or another archive. -Looking at an output file, e.g. `/tmp/jdeps330/spark3xx-common.dot`, +Looking at an output file, e.g. `/tmp/jdeps330/spark-shared.dot`, unfortunately you see that jdeps does not label the source class node but labels the target class node of an edge. Thus the graph is incorrect as it breaks paths if a node has both incoming and outgoing edges. ```bash -$ grep 'com.nvidia.spark.rapids.GpuFilterExec\$' spark3xx-common.dot +$ grep 'com.nvidia.spark.rapids.GpuFilterExec\$' spark-shared.dot "com.nvidia.spark.rapids.GpuFilterExec$" -> "com.nvidia.spark.rapids.GpuFilterExec (spark330)"; - "com.nvidia.spark.rapids.GpuOverrides$$anon$204" -> "com.nvidia.spark.rapids.GpuFilterExec$ (spark3xx-common)"; + "com.nvidia.spark.rapids.GpuOverrides$$anon$204" -> "com.nvidia.spark.rapids.GpuFilterExec$ (spark-shared)"; ``` So first create and `cd` to some other directory `/tmp/jdep330.processed` to massage @@ -214,8 +214,8 @@ that the source nodes are guaranteed to be from the ``. ```bash sed 's/"\([^(]*\)"\(\s*->.*;\)/"\1 (public)"\2/' \ /tmp/jdeps330/public.dot > public.dot -sed 's/"\([^(]*\)"\(\s*->.*;\)/"\1 (spark3xx-common)"\2/' \ - /tmp/jdeps330/spark3xx-common.dot > spark3xx-common.dot +sed 's/"\([^(]*\)"\(\s*->.*;\)/"\1 (spark-shared)"\2/' \ + /tmp/jdeps330/spark-shared.dot > spark-shared.dot sed 's/"\([^(]*\)"\(\s*->.*;\)/"\1 (spark330)"\2/' \ /tmp/jdeps330/spark330.dot > spark330.dot ``` @@ -224,7 +224,7 @@ Next you need to union edges of all three graphs into a single graph to be able to analyze cross-archive paths. ```bash -cat public.dot spark3xx-common.dot spark330.dot | \ +cat public.dot spark-shared.dot spark330.dot | \ tr '\n' '\r' | \ sed 's/}\rdigraph "[^"]*" {\r//g' | \ tr '\r' '\n' > merged.dot @@ -245,7 +245,7 @@ GpuTypeColumnVector needs refactoring prior externalization as of the time of this writing: ```bash -$ dijkstra -d -p "com.nvidia.spark.rapids.GpuColumnVector (spark3xx-common)" merged.dot | \ +$ dijkstra -d -p "com.nvidia.spark.rapids.GpuColumnVector (spark-shared)" merged.dot | \ grep '\[dist=' | grep '(spark330)' "org.apache.spark.sql.rapids.GpuFileSourceScanExec (spark330)" [dist=5.000, "com.nvidia.spark.rapids.GpuExec (spark330)" [dist=3.000, @@ -255,9 +255,9 @@ $ dijkstra -d -p "com.nvidia.spark.rapids.GpuColumnVector (spark3xx-common)" mer RegexReplace could be externalized safely: ```bash -$ dijkstra -d -p "org.apache.spark.sql.rapids.RegexReplace (spark3xx-common)" merged.dot | grep '\[dist=' - "org.apache.spark.sql.rapids.RegexReplace (spark3xx-common)" [dist=0.000]; - "org.apache.spark.sql.rapids.RegexReplace$ (spark3xx-common)" [dist=1.000, +$ dijkstra -d -p "org.apache.spark.sql.rapids.RegexReplace (spark-shared)" merged.dot | grep '\[dist=' + "org.apache.spark.sql.rapids.RegexReplace (spark-shared)" [dist=0.000]; + "org.apache.spark.sql.rapids.RegexReplace$ (spark-shared)" [dist=1.000, ``` because it is self-contained. diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh index 697722c0138..ec4b74cc902 100755 --- a/jenkins/spark-premerge-build.sh +++ b/jenkins/spark-premerge-build.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -88,11 +88,11 @@ mvn_verify() { FILE=$(ls dist/target/rapids-4-spark_2.12-*.jar | grep -v test | xargs readlink -f) UDF_JAR=$(ls ./udf-compiler/target/spark${SPK_VER}/rapids-4-spark-udf_2.12-*-spark${SPK_VER}.jar | grep -v test | xargs readlink -f) pushd target/jacoco_classes/ - jar xf $FILE com org rapids spark3xx-common "spark${JACOCO_SPARK_VER:-311}/" + jar xf $FILE com org rapids spark-shared "spark${JACOCO_SPARK_VER:-311}/" # extract the .class files in udf jar and replace the existing ones in spark3xx-ommon and spark$SPK_VER # because the class files in udf jar will be modified in aggregator's shade phase jar xf "$UDF_JAR" com/nvidia/spark/udf - rm -rf com/nvidia/shaded/ org/openucx/ spark3xx-common/com/nvidia/spark/udf/ spark${SPK_VER}/com/nvidia/spark/udf/ + rm -rf com/nvidia/shaded/ org/openucx/ spark-shared/com/nvidia/spark/udf/ spark${SPK_VER}/com/nvidia/spark/udf/ popd # Triggering here until we change the jenkins file diff --git a/scala2.13/delta-lake/delta-20x/pom.xml b/scala2.13/delta-lake/delta-20x/pom.xml index 6b098f9bdb1..40547c3f510 100644 --- a/scala2.13/delta-lake/delta-20x/pom.xml +++ b/scala2.13/delta-lake/delta-20x/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/scala2.13/delta-lake/delta-21x/pom.xml b/scala2.13/delta-lake/delta-21x/pom.xml index 6f58d97b9c9..0f228709016 100644 --- a/scala2.13/delta-lake/delta-21x/pom.xml +++ b/scala2.13/delta-lake/delta-21x/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/scala2.13/delta-lake/delta-22x/pom.xml b/scala2.13/delta-lake/delta-22x/pom.xml index 500405c9f1a..c315b05ad36 100644 --- a/scala2.13/delta-lake/delta-22x/pom.xml +++ b/scala2.13/delta-lake/delta-22x/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/scala2.13/delta-lake/delta-23x/pom.xml b/scala2.13/delta-lake/delta-23x/pom.xml index c6e9cf634a6..7ac7c1156dd 100644 --- a/scala2.13/delta-lake/delta-23x/pom.xml +++ b/scala2.13/delta-lake/delta-23x/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/scala2.13/delta-lake/delta-24x/pom.xml b/scala2.13/delta-lake/delta-24x/pom.xml index 987d24b0172..2d1a1b43e82 100644 --- a/scala2.13/delta-lake/delta-24x/pom.xml +++ b/scala2.13/delta-lake/delta-24x/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/scala2.13/delta-lake/delta-spark330db/pom.xml b/scala2.13/delta-lake/delta-spark330db/pom.xml index b5f5a9e475b..46d6f8798d9 100644 --- a/scala2.13/delta-lake/delta-spark330db/pom.xml +++ b/scala2.13/delta-lake/delta-spark330db/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/scala2.13/delta-lake/delta-spark332db/pom.xml b/scala2.13/delta-lake/delta-spark332db/pom.xml index 2ea75281f26..93a942e9d44 100644 --- a/scala2.13/delta-lake/delta-spark332db/pom.xml +++ b/scala2.13/delta-lake/delta-spark332db/pom.xml @@ -39,6 +39,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/scala2.13/delta-lake/delta-spark341db/pom.xml b/scala2.13/delta-lake/delta-spark341db/pom.xml index 30c513c6f3d..6b5e545291b 100644 --- a/scala2.13/delta-lake/delta-spark341db/pom.xml +++ b/scala2.13/delta-lake/delta-spark341db/pom.xml @@ -38,6 +38,10 @@ + + org.roaringbitmap + RoaringBitmap + com.nvidia rapids-4-spark-sql_${scala.binary.version} diff --git a/scala2.13/sql-plugin/pom.xml b/scala2.13/sql-plugin/pom.xml index eb6f240a3f6..df3532a3592 100644 --- a/scala2.13/sql-plugin/pom.xml +++ b/scala2.13/sql-plugin/pom.xml @@ -97,10 +97,6 @@ org.alluxio alluxio-shaded-client - - org.roaringbitmap - RoaringBitmap - org.mockito mockito-core diff --git a/sql-plugin/pom.xml b/sql-plugin/pom.xml index 08657a9d40b..961e6f08372 100644 --- a/sql-plugin/pom.xml +++ b/sql-plugin/pom.xml @@ -97,10 +97,6 @@ org.alluxio alluxio-shaded-client - - org.roaringbitmap - RoaringBitmap - org.mockito mockito-core From 7dc52bcefea19ebfa5a3c7985205a4e5e7fc9122 Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Fri, 28 Jun 2024 09:07:31 -0700 Subject: [PATCH 50/79] Fixed some cast_tests (#11049) Signed-off-by: Raza Jafri --- integration_tests/src/main/python/cast_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/integration_tests/src/main/python/cast_test.py b/integration_tests/src/main/python/cast_test.py index 2621124b4b8..a56afe29ab9 100644 --- a/integration_tests/src/main/python/cast_test.py +++ b/integration_tests/src/main/python/cast_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -427,7 +427,10 @@ def fun(spark): data = [invalid_value] df = spark.createDataFrame(data, type) return df.select(f.col('value').cast(TimestampType())).collect() - assert_gpu_and_cpu_error(fun, {"spark.sql.ansi.enabled": True}, "SparkDateTimeException") + + assert_gpu_and_cpu_error(fun, {"spark.sql.ansi.enabled": True}, + error_message="SparkDateTimeException" + if is_before_spark_400() else "DateTimeException") # if float.floor > Long.max or float.ceil < Long.min, throw exception @pytest.mark.skipif(is_before_spark_330(), reason="ansi cast throws exception only in 3.3.0+") @@ -583,7 +586,7 @@ def fun(spark): data=[invalid_string] df = spark.createDataFrame(data, StringType()) return df.select(f.col('value').cast(dtType)).collect() - assert_gpu_and_cpu_error(fun, {}, "java.lang.IllegalArgumentException") + assert_gpu_and_cpu_error(fun, {}, "IllegalArgumentException") @pytest.mark.skipif(is_before_spark_330(), reason='casting between interval and integral is not supported before Pyspark 3.3.0') def test_cast_day_time_interval_to_integral_no_overflow(): From dd62000d91d7563b20a1af8b1e015a81bd056061 Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Fri, 28 Jun 2024 09:31:36 -0700 Subject: [PATCH 51/79] Fixed array_tests for Spark 4.0.0 [databricks] (#11048) * Fixed array_tests * Signing off Signed-off-by: Raza Jafri * Disable ANSI for failing tests --------- Signed-off-by: Raza Jafri --- .../src/main/python/array_test.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/integration_tests/src/main/python/array_test.py b/integration_tests/src/main/python/array_test.py index 0b98bd23439..f3d84aff372 100644 --- a/integration_tests/src/main/python/array_test.py +++ b/integration_tests/src/main/python/array_test.py @@ -17,8 +17,8 @@ from asserts import assert_gpu_and_cpu_are_equal_collect, assert_gpu_and_cpu_are_equal_sql, assert_gpu_and_cpu_error, assert_gpu_fallback_collect from data_gen import * from conftest import is_databricks_runtime -from marks import incompat, allow_non_gpu -from spark_session import is_before_spark_313, is_before_spark_330, is_databricks113_or_later, is_spark_330_or_later, is_databricks104_or_later, is_spark_33X, is_spark_340_or_later, is_spark_330, is_spark_330cdh +from marks import incompat, allow_non_gpu, disable_ansi_mode +from spark_session import * from pyspark.sql.types import * from pyspark.sql.types import IntegralType from pyspark.sql.functions import array_contains, col, element_at, lit, array @@ -103,11 +103,13 @@ @pytest.mark.parametrize('data_gen', array_item_test_gens, ids=idfn) @pytest.mark.parametrize('index_gen', array_index_gens, ids=idfn) +@disable_ansi_mode def test_array_item(data_gen, index_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark: two_col_df(spark, data_gen, index_gen).selectExpr('a[b]')) @pytest.mark.parametrize('data_gen', array_item_test_gens, ids=idfn) +@disable_ansi_mode def test_array_item_lit_ordinal(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark: unary_op_df(spark, data_gen).selectExpr( @@ -145,8 +147,10 @@ def test_array_item_with_strict_index(strict_index_enabled, index): # No need to test this for multiple data types for array. Only one is enough, but with two kinds of invalid index. @pytest.mark.parametrize('index', [-2, 100, array_neg_index_gen, array_out_index_gen], ids=idfn) +@disable_ansi_mode def test_array_item_ansi_fail_invalid_index(index): - message = "SparkArrayIndexOutOfBoundsException" if (is_databricks104_or_later() or is_spark_330_or_later()) else "java.lang.ArrayIndexOutOfBoundsException" + message = "SparkArrayIndexOutOfBoundsException" if (is_databricks104_or_later() or is_spark_330_or_later() and is_before_spark_400()) else \ + "ArrayIndexOutOfBoundsException" if isinstance(index, int): test_func = lambda spark: unary_op_df(spark, ArrayGen(int_gen)).select(col('a')[index]).collect() else: @@ -171,6 +175,7 @@ def test_array_item_ansi_not_fail_all_null_data(): decimal_gen_32bit, decimal_gen_64bit, decimal_gen_128bit, binary_gen, StructGen([['child0', StructGen([['child01', IntegerGen()]])], ['child1', string_gen], ['child2', float_gen]], nullable=False), StructGen([['child0', byte_gen], ['child1', string_gen], ['child2', float_gen]], nullable=False)], ids=idfn) +@disable_ansi_mode def test_make_array(data_gen): (s1, s2) = with_cpu_session( lambda spark: gen_scalars_for_sql(data_gen, 2, force_no_nulls=not isinstance(data_gen, NullGen))) @@ -212,6 +217,7 @@ def test_orderby_array_of_structs(data_gen): @pytest.mark.parametrize('data_gen', [byte_gen, short_gen, int_gen, long_gen, float_gen, double_gen, string_gen, boolean_gen, date_gen, timestamp_gen], ids=idfn) +@disable_ansi_mode def test_array_contains(data_gen): arr_gen = ArrayGen(data_gen) literal = with_cpu_session(lambda spark: gen_scalar(data_gen, force_no_nulls=True)) @@ -239,6 +245,7 @@ def test_array_contains_for_nans(data_gen): @pytest.mark.parametrize('data_gen', array_item_test_gens, ids=idfn) +@disable_ansi_mode def test_array_element_at(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark: two_col_df(spark, data_gen, array_no_zero_index_gen).selectExpr( @@ -252,8 +259,9 @@ def test_array_element_at(data_gen): # No need tests for multiple data types for list data. Only one is enough. @pytest.mark.parametrize('index', [100, array_out_index_gen], ids=idfn) +@disable_ansi_mode def test_array_element_at_ansi_fail_invalid_index(index): - message = "ArrayIndexOutOfBoundsException" if is_before_spark_330() else "SparkArrayIndexOutOfBoundsException" + message = "ArrayIndexOutOfBoundsException" if is_before_spark_330() or not is_before_spark_400() else "SparkArrayIndexOutOfBoundsException" if isinstance(index, int): test_func = lambda spark: unary_op_df(spark, ArrayGen(int_gen)).select( element_at(col('a'), index)).collect() @@ -282,9 +290,10 @@ def test_array_element_at_ansi_not_fail_all_null_data(): @pytest.mark.parametrize('index', [0, array_zero_index_gen], ids=idfn) @pytest.mark.parametrize('ansi_enabled', [False, True], ids=idfn) +@disable_ansi_mode def test_array_element_at_zero_index_fail(index, ansi_enabled): if is_spark_340_or_later(): - message = "org.apache.spark.SparkRuntimeException: [INVALID_INDEX_OF_ZERO] The index 0 is invalid" + message = "SparkRuntimeException: [INVALID_INDEX_OF_ZERO] The index 0 is invalid" elif is_databricks113_or_later(): message = "org.apache.spark.SparkRuntimeException: [ELEMENT_AT_BY_INDEX_ZERO] The index 0 is invalid" else: @@ -303,6 +312,7 @@ def test_array_element_at_zero_index_fail(index, ansi_enabled): @pytest.mark.parametrize('data_gen', array_gens_sample, ids=idfn) +@disable_ansi_mode def test_array_transform(data_gen): def do_it(spark): columns = ['a', 'b', From f95402640fb5b85ae96078d14b358aef4424ef04 Mon Sep 17 00:00:00 2001 From: "Hongbin Ma (Mahone)" Date: Sat, 29 Jun 2024 09:02:27 +0800 Subject: [PATCH 52/79] Add a heuristic to skip second or third agg pass (#10950) * add a heristic to skip agg pass Signed-off-by: Hongbin Ma (Mahone) * commit doc change Signed-off-by: Hongbin Ma (Mahone) * refine naming Signed-off-by: Hongbin Ma (Mahone) * fix only reduction case Signed-off-by: Hongbin Ma (Mahone) * fix compile Signed-off-by: Hongbin Ma (Mahone) * fix Signed-off-by: Hongbin Ma (Mahone) * clean Signed-off-by: Hongbin Ma (Mahone) * fix doc Signed-off-by: Hongbin Ma (Mahone) * reduce premergeci2 Signed-off-by: Hongbin Ma (Mahone) * reduce premergeci2, 2 Signed-off-by: Hongbin Ma (Mahone) * use test_parallel to workaround flaky array test Signed-off-by: Hongbin Ma (Mahone) * address review comment Signed-off-by: Hongbin Ma (Mahone) * remove comma Signed-off-by: Hongbin Ma (Mahone) * workaround for ci_scala213 Signed-off-by: Hongbin Ma (Mahone) * disable agg ratio heruistic by default Signed-off-by: Hongbin Ma (Mahone) * fix doc Signed-off-by: Hongbin Ma (Mahone) --------- Signed-off-by: Hongbin Ma (Mahone) --- .../advanced_configs.md | 1 + .../src/main/python/hash_aggregate_test.py | 11 +- .../spark/rapids/GpuAggregateExec.scala | 130 ++++++++++++++---- .../com/nvidia/spark/rapids/RapidsConf.scala | 10 ++ 4 files changed, 124 insertions(+), 28 deletions(-) diff --git a/docs/additional-functionality/advanced_configs.md b/docs/additional-functionality/advanced_configs.md index 941ab4046e6..033e332b99c 100644 --- a/docs/additional-functionality/advanced_configs.md +++ b/docs/additional-functionality/advanced_configs.md @@ -60,6 +60,7 @@ Name | Description | Default Value | Applicable at spark.rapids.shuffle.ucx.activeMessages.forceRndv|Set to true to force 'rndv' mode for all UCX Active Messages. This should only be required with UCX 1.10.x. UCX 1.11.x deployments should set to false.|false|Startup spark.rapids.shuffle.ucx.managementServerHost|The host to be used to start the management server|null|Startup spark.rapids.shuffle.ucx.useWakeup|When set to true, use UCX's event-based progress (epoll) in order to wake up the progress thread when needed, instead of a hot loop.|true|Startup +spark.rapids.sql.agg.skipAggPassReductionRatio|In non-final aggregation stages, if the previous pass has a row reduction ratio greater than this value, the next aggregation pass will be skipped.Setting this to 1 essentially disables this feature.|1.0|Runtime spark.rapids.sql.allowMultipleJars|Allow multiple rapids-4-spark, spark-rapids-jni, and cudf jars on the classpath. Spark will take the first one it finds, so the version may not be expected. Possisble values are ALWAYS: allow all jars, SAME_REVISION: only allow jars with the same revision, NEVER: do not allow multiple jars at all.|SAME_REVISION|Startup spark.rapids.sql.castDecimalToFloat.enabled|Casting from decimal to floating point types on the GPU returns results that have tiny difference compared to results returned from CPU.|true|Runtime spark.rapids.sql.castFloatToDecimal.enabled|Casting from floating point types to decimal on the GPU returns results that have tiny difference compared to results returned from CPU.|true|Runtime diff --git a/integration_tests/src/main/python/hash_aggregate_test.py b/integration_tests/src/main/python/hash_aggregate_test.py index e24a34ef3d5..d1cd70aa43c 100644 --- a/integration_tests/src/main/python/hash_aggregate_test.py +++ b/integration_tests/src/main/python/hash_aggregate_test.py @@ -29,12 +29,15 @@ pytestmark = pytest.mark.nightly_resource_consuming_test _float_conf = {'spark.rapids.sql.variableFloatAgg.enabled': 'true', - 'spark.rapids.sql.castStringToFloat.enabled': 'true' - } + 'spark.rapids.sql.castStringToFloat.enabled': 'true' + } _float_smallbatch_conf = copy_and_update(_float_conf, {'spark.rapids.sql.batchSizeBytes' : '250'}) +_float_conf_skipagg = copy_and_update(_float_smallbatch_conf, + {'spark.rapids.sql.agg.skipAggPassReductionRatio': '0'}) + _float_conf_partial = copy_and_update(_float_conf, {'spark.rapids.sql.hashAgg.replaceMode': 'partial'}) @@ -221,8 +224,8 @@ def get_params(init_list, marked_params=[]): return list -# Run these tests with in 4 modes, all on the GPU -_confs = [_float_conf, _float_smallbatch_conf, _float_conf_final, _float_conf_partial] +# Run these tests with in 5 modes, all on the GPU +_confs = [_float_conf, _float_smallbatch_conf, _float_conf_skipagg, _float_conf_final, _float_conf_partial] # Pytest marker for list of operators allowed to run on the CPU, # esp. useful in partial and final only modes. diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuAggregateExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuAggregateExec.scala index c58d9862be1..7e6a1056d01 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuAggregateExec.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuAggregateExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ package com.nvidia.spark.rapids import java.util import scala.annotation.tailrec +import scala.collection.JavaConverters.collectionAsScalaIterableConverter import scala.collection.mutable import ai.rapids.cudf @@ -549,7 +550,8 @@ object GpuAggregateIterator extends Logging { object GpuAggFirstPassIterator { def apply(cbIter: Iterator[ColumnarBatch], aggHelper: AggHelper, - metrics: GpuHashAggregateMetrics): Iterator[SpillableColumnarBatch] = { + metrics: GpuHashAggregateMetrics + ): Iterator[SpillableColumnarBatch] = { val preprocessProjectIter = cbIter.map { cb => val sb = SpillableColumnarBatch (cb, SpillPriorities.ACTIVE_ON_DECK_PRIORITY) aggHelper.preStepBound.projectAndCloseWithRetrySingleBatch (sb) @@ -707,6 +709,9 @@ object GpuAggFinalPassIterator { * @param metrics metrics that will be updated during aggregation * @param configuredTargetBatchSize user-specified value for the targeted input batch size * @param useTieredProject user-specified option to enable tiered projections + * @param allowNonFullyAggregatedOutput if allowed to skip third pass Agg + * @param skipAggPassReductionRatio skip if the ratio of rows after a pass is bigger than this value + * @param localInputRowsCount metric to track the number of input rows processed locally */ class GpuMergeAggregateIterator( firstPassIter: Iterator[SpillableColumnarBatch], @@ -718,15 +723,21 @@ class GpuMergeAggregateIterator( modeInfo: AggregateModeInfo, metrics: GpuHashAggregateMetrics, configuredTargetBatchSize: Long, - useTieredProject: Boolean) + useTieredProject: Boolean, + allowNonFullyAggregatedOutput: Boolean, + skipAggPassReductionRatio: Double, + localInputRowsCount: LocalGpuMetric) extends Iterator[ColumnarBatch] with AutoCloseable with Logging { private[this] val isReductionOnly = groupingExpressions.isEmpty private[this] val targetMergeBatchSize = computeTargetMergeBatchSize(configuredTargetBatchSize) private[this] val aggregatedBatches = new util.ArrayDeque[SpillableColumnarBatch] private[this] var outOfCoreIter: Option[GpuOutOfCoreSortIterator] = None - /** Iterator for fetching aggregated batches if a sort-based fallback has occurred */ - private[this] var sortFallbackIter: Option[Iterator[ColumnarBatch]] = None + /** Iterator for fetching aggregated batches either if: + * 1. a sort-based fallback has occurred + * 2. skip third pass agg has occurred + **/ + private[this] var fallbackIter: Option[Iterator[ColumnarBatch]] = None /** Whether a batch is pending for a reduction-only aggregation */ private[this] var hasReductionOnlyBatch: Boolean = isReductionOnly @@ -739,24 +750,61 @@ class GpuMergeAggregateIterator( } override def hasNext: Boolean = { - sortFallbackIter.map(_.hasNext).getOrElse { + fallbackIter.map(_.hasNext).getOrElse { // reductions produce a result even if the input is empty hasReductionOnlyBatch || !aggregatedBatches.isEmpty || firstPassIter.hasNext } } override def next(): ColumnarBatch = { - sortFallbackIter.map(_.next()).getOrElse { + fallbackIter.map(_.next()).getOrElse { + var shouldSkipThirdPassAgg = false + // aggregate and merge all pending inputs if (firstPassIter.hasNext) { - aggregateInputBatches() - tryMergeAggregatedBatches() + // first pass agg + val rowsAfterFirstPassAgg = aggregateInputBatches() + + // by now firstPassIter has been traversed, so localInputRowsCount is finished updating + if (isReductionOnly || + skipAggPassReductionRatio * localInputRowsCount.value >= rowsAfterFirstPassAgg) { + // second pass agg + tryMergeAggregatedBatches() + + val rowsAfterSecondPassAgg = aggregatedBatches.asScala.foldLeft(0L) { + (totalRows, batch) => totalRows + batch.numRows() + } + shouldSkipThirdPassAgg = + rowsAfterSecondPassAgg > skipAggPassReductionRatio * rowsAfterFirstPassAgg + } else { + shouldSkipThirdPassAgg = true + logInfo(s"Rows after first pass aggregation $rowsAfterFirstPassAgg exceeds " + + s"${skipAggPassReductionRatio * 100}% of " + + s"localInputRowsCount ${localInputRowsCount.value}, skip the second pass agg") + } } if (aggregatedBatches.size() > 1) { - // Unable to merge to a single output, so must fall back to a sort-based approach. - sortFallbackIter = Some(buildSortFallbackIterator()) - sortFallbackIter.get.next() + // Unable to merge to a single output, so must fall back + if (allowNonFullyAggregatedOutput && shouldSkipThirdPassAgg) { + // skip third pass agg, return the aggregated batches directly + logInfo(s"Rows after second pass aggregation exceeds " + + s"${skipAggPassReductionRatio * 100}% of " + + s"rows after first pass, skip the third pass agg") + fallbackIter = Some(new Iterator[ColumnarBatch] { + override def hasNext: Boolean = !aggregatedBatches.isEmpty + + override def next(): ColumnarBatch = { + withResource(aggregatedBatches.pop()) { spillableBatch => + spillableBatch.getColumnarBatch() + } + } + }) + } else { + // fallback to sort agg, this is the third pass agg + fallbackIter = Some(buildSortFallbackIterator()) + } + fallbackIter.get.next() } else if (aggregatedBatches.isEmpty) { if (hasReductionOnlyBatch) { hasReductionOnlyBatch = false @@ -779,7 +827,7 @@ class GpuMergeAggregateIterator( aggregatedBatches.clear() outOfCoreIter.foreach(_.close()) outOfCoreIter = None - sortFallbackIter = None + fallbackIter = None hasReductionOnlyBatch = false } @@ -789,11 +837,15 @@ class GpuMergeAggregateIterator( } /** Aggregate all input batches and place the results in the aggregatedBatches queue. */ - private def aggregateInputBatches(): Unit = { + private def aggregateInputBatches(): Long = { + var rowsAfter = 0L // cache everything in the first pass while (firstPassIter.hasNext) { - aggregatedBatches.add(firstPassIter.next()) + val batch = firstPassIter.next() + rowsAfter += batch.numRows() + aggregatedBatches.add(batch) } + rowsAfter } /** @@ -1115,8 +1167,8 @@ abstract class GpuBaseAggregateMeta[INPUT <: SparkPlan]( /* * Type inferencing by the Scala compiler will choose the most specific return type - * something like Array[Set[Product with Serializable with AggregateMode]] or with - * slight differences depending on Scala version. Here we ensure this is + * something like Array[Set[Product with Serializable with AggregateMode]] or with + * slight differences depending on Scala version. Here we ensure this is * Array[Set[AggregateMode]] to perform the subsequent Set and Array operations properly. */ val aggPatternsCanReplace = strPatternToReplace.split("\\|").map { subPattern => @@ -1189,6 +1241,12 @@ abstract class GpuBaseAggregateMeta[INPUT <: SparkPlan]( mode == Partial || mode == PartialMerge } && agg.groupingExpressions.nonEmpty // Don't do this for a reduce... + // for a aggregateExpressions.isEmpty case, we cannot distinguish between final and non-final, + // so don't allow it. + lazy val allowNonFullyAggregatedOutput = aggModes.forall { mode => + mode == Partial || mode == PartialMerge + } && agg.aggregateExpressions.nonEmpty + lazy val groupingCanBeSorted = agg.groupingExpressions.forall { expr => orderable.isSupportedByPlugin(expr.dataType) } @@ -1272,7 +1330,9 @@ abstract class GpuBaseAggregateMeta[INPUT <: SparkPlan]( useTiered, estimatedPreProcessGrowth, conf.forceSinglePassPartialSortAgg, - allowSinglePassAgg) + allowSinglePassAgg, + allowNonFullyAggregatedOutput, + conf.skipAggPassReductionRatio) } } @@ -1358,7 +1418,9 @@ abstract class GpuTypedImperativeSupportedAggregateExecMeta[INPUT <: BaseAggrega // For now we are just going to go with the original hash aggregation 1.0, false, - false) + false, + false, + 1) } else { super.convertToGpu() } @@ -1707,6 +1769,10 @@ object GpuHashAggregateExecBase { * @param child incoming plan (where we get input columns from) * @param configuredTargetBatchSize user-configured maximum device memory size of a batch * @param configuredTieredProjectEnabled configurable optimization to use tiered projections + * @param allowNonFullyAggregatedOutput whether we can skip the third pass of aggregation + * (can omit non fully aggregated data for non-final + * stage of aggregation) + * @param skipAggPassReductionRatio skip if the ratio of rows after a pass is bigger than this value */ case class GpuHashAggregateExec( requiredChildDistributionExpressions: Option[Seq[Expression]], @@ -1719,7 +1785,10 @@ case class GpuHashAggregateExec( configuredTieredProjectEnabled: Boolean, estimatedPreProcessGrowth: Double, forceSinglePassAgg: Boolean, - allowSinglePassAgg: Boolean) extends ShimUnaryExecNode with GpuExec { + allowSinglePassAgg: Boolean, + allowNonFullyAggregatedOutput: Boolean, + skipAggPassReductionRatio: Double +) extends ShimUnaryExecNode with GpuExec { // lifted directly from `BaseAggregateExec.inputAttributes`, edited comment. def inputAttributes: Seq[Attribute] = @@ -1804,7 +1873,7 @@ case class GpuHashAggregateExec( boundGroupExprs, aggregateExprs, aggregateAttrs, resultExprs, modeInfo, localEstimatedPreProcessGrowth, alreadySorted, expectedOrdering, postBoundReferences, targetBatchSize, aggMetrics, useTieredProject, - localForcePre, localAllowPre) + localForcePre, localAllowPre, allowNonFullyAggregatedOutput, skipAggPassReductionRatio) } } @@ -1920,7 +1989,10 @@ class DynamicGpuPartialSortAggregateIterator( metrics: GpuHashAggregateMetrics, useTiered: Boolean, forceSinglePassAgg: Boolean, - allowSinglePassAgg: Boolean) extends Iterator[ColumnarBatch] { + allowSinglePassAgg: Boolean, + allowNonFullyAggregatedOutput: Boolean, + skipAggPassReductionRatio: Double +) extends Iterator[ColumnarBatch] { private var aggIter: Option[Iterator[ColumnarBatch]] = None private[this] val isReductionOnly = boundGroupExprs.outputTypes.isEmpty @@ -1998,7 +2070,14 @@ class DynamicGpuPartialSortAggregateIterator( inputAttrs.map(_.dataType).toArray, preProcessAggHelper.preStepBound, metrics.opTime, metrics.numPreSplits) - val firstPassIter = GpuAggFirstPassIterator(splitInputIter, preProcessAggHelper, metrics) + val localInputRowsMetrics = new LocalGpuMetric + val firstPassIter = GpuAggFirstPassIterator( + splitInputIter.map(cb => { + localInputRowsMetrics += cb.numRows() + cb + }), + preProcessAggHelper, + metrics) val mergeIter = new GpuMergeAggregateIterator( firstPassIter, @@ -2010,7 +2089,10 @@ class DynamicGpuPartialSortAggregateIterator( modeInfo, metrics, configuredTargetBatchSize, - useTiered) + useTiered, + allowNonFullyAggregatedOutput, + skipAggPassReductionRatio, + localInputRowsMetrics) GpuAggFinalPassIterator.makeIter(mergeIter, postBoundReferences, metrics) } diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index 5203e926efa..aad4f05b334 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -1509,6 +1509,14 @@ val GPU_COREDUMP_PIPE_PATTERN = conf("spark.rapids.gpu.coreDump.pipePattern") .booleanConf .createWithDefault(true) + val SKIP_AGG_PASS_REDUCTION_RATIO = conf("spark.rapids.sql.agg.skipAggPassReductionRatio") + .doc("In non-final aggregation stages, if the previous pass has a row reduction ratio " + + "greater than this value, the next aggregation pass will be skipped." + + "Setting this to 1 essentially disables this feature.") + .doubleConf + .checkValue(v => v >= 0 && v <= 1, "The ratio value must be in [0, 1].") + .createWithDefault(1.0) + val FORCE_SINGLE_PASS_PARTIAL_SORT_AGG: ConfEntryWithDefault[Boolean] = conf("spark.rapids.sql.agg.forceSinglePassPartialSort") .doc("Force a single pass partial sort agg to happen in all cases that it could, " + @@ -3069,6 +3077,8 @@ class RapidsConf(conf: Map[String, String]) extends Logging { lazy val forceSinglePassPartialSortAgg: Boolean = get(FORCE_SINGLE_PASS_PARTIAL_SORT_AGG) + lazy val skipAggPassReductionRatio: Double = get(SKIP_AGG_PASS_REDUCTION_RATIO) + lazy val isRegExpEnabled: Boolean = get(ENABLE_REGEXP) lazy val maxRegExpStateMemory: Long = { From 2498204313dbc2f21cbdd76f84d5b92068949620 Mon Sep 17 00:00:00 2001 From: Haoyang Li Date: Sat, 29 Jun 2024 10:06:05 +0800 Subject: [PATCH 53/79] Support regex patterns with brackets when rewriting to PrefixRange pattern in rlike. (#11088) * Remove bracket when necessary in PrefixRange patten in Regex rewrite Signed-off-by: Haoyang Li * add pytest cases Signed-off-by: Haoyang Li * fix scala 2.13 build Signed-off-by: Haoyang Li --------- Signed-off-by: Haoyang Li --- .../src/main/python/regexp_test.py | 2 + .../com/nvidia/spark/rapids/RegexParser.scala | 45 ++++++++++--------- .../RegularExpressionRewriteSuite.scala | 4 ++ 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/integration_tests/src/main/python/regexp_test.py b/integration_tests/src/main/python/regexp_test.py index 18a83870d83..c2062605ca1 100644 --- a/integration_tests/src/main/python/regexp_test.py +++ b/integration_tests/src/main/python/regexp_test.py @@ -468,6 +468,8 @@ def test_rlike_rewrite_optimization(): 'rlike(a, "a[a-c]{1,3}")', 'rlike(a, "a[a-c]{1,}")', 'rlike(a, "a[a-c]+")', + 'rlike(a, "(ab)([a-c]{1})")', + 'rlike(a, "(ab[a-c]{1})")', 'rlike(a, "(aaa|bbb|ccc)")', 'rlike(a, ".*.*(aaa|bbb).*.*")', 'rlike(a, "^.*(aaa|bbb|ccc)")', diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala index 1ca155f8a52..362a9cce293 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala @@ -2035,7 +2035,7 @@ object RegexRewrite { @scala.annotation.tailrec private def removeBrackets(astLs: collection.Seq[RegexAST]): collection.Seq[RegexAST] = { astLs match { - case collection.Seq(RegexGroup(_, term, None)) => removeBrackets(term.children()) + case collection.Seq(RegexGroup(_, RegexSequence(terms), None)) => removeBrackets(terms) case _ => astLs } } @@ -2051,28 +2051,31 @@ object RegexRewrite { Option[(String, Int, Int, Int)] = { val haveLiteralPrefix = isLiteralString(astLs.dropRight(1)) val endsWithRange = astLs.lastOption match { - case Some(RegexRepetition( - RegexCharacterClass(false, ListBuffer(RegexCharacterRange(a,b))), - quantifier)) => { - val (start, end) = (a, b) match { - case (RegexChar(start), RegexChar(end)) => (start, end) - case _ => return None - } - val length = quantifier match { - // In Rlike, contains [a-b]{minLen,maxLen} pattern is equivalent to contains - // [a-b]{minLen} because the matching will return the result once it finds the - // minimum match so y here is unnecessary. - case QuantifierVariableLength(minLen, _) => minLen - case QuantifierFixedLength(len) => len - case SimpleQuantifier(ch) => ch match { - case '*' | '?' => 0 - case '+' => 1 + case Some(ast) => removeBrackets(collection.Seq(ast)) match { + case collection.Seq(RegexRepetition( + RegexCharacterClass(false, ListBuffer(RegexCharacterRange(a,b))), + quantifier)) => { + val (start, end) = (a, b) match { + case (RegexChar(start), RegexChar(end)) => (start, end) + case _ => return None + } + val length = quantifier match { + // In Rlike, contains [a-b]{minLen,maxLen} pattern is equivalent to contains + // [a-b]{minLen} because the matching will return the result once it finds the + // minimum match so y here is unnecessary. + case QuantifierVariableLength(minLen, _) => minLen + case QuantifierFixedLength(len) => len + case SimpleQuantifier(ch) => ch match { + case '*' | '?' => 0 + case '+' => 1 + case _ => return None + } case _ => return None } - case _ => return None + // Convert start and end to code points + Some((length, start.toInt, end.toInt)) } - // Convert start and end to code points - Some((length, start.toInt, end.toInt)) + case _ => None } case _ => None } @@ -2153,7 +2156,7 @@ object RegexRewrite { } } - val noStartsWithAst = stripLeadingWildcards(noTailingWildcards) + val noStartsWithAst = removeBrackets(stripLeadingWildcards(noTailingWildcards)) // Check if the pattern is a contains literal pattern if (isLiteralString(noStartsWithAst)) { diff --git a/tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionRewriteSuite.scala b/tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionRewriteSuite.scala index 7626c1450c1..a55815b95ef 100644 --- a/tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionRewriteSuite.scala +++ b/tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionRewriteSuite.scala @@ -52,6 +52,8 @@ class RegularExpressionRewriteSuite extends AnyFunSuite { "(.*)abc[0-9]{1,3}(.*)", "(.*)abc[0-9a-z]{1,3}(.*)", "(.*)abc[0-9]{2}.*", + "((abc))([0-9]{3})", + "(abc[0-9]{3})", "^abc[0-9]{1,3}", "火花急流[\u4e00-\u9fa5]{1}", "^[0-9]{6}", @@ -63,6 +65,8 @@ class RegularExpressionRewriteSuite extends AnyFunSuite { PrefixRange("abc", 1, 48, 57), NoOptimization, // prefix followed by a multi-range not supported PrefixRange("abc", 2, 48, 57), + PrefixRange("abc", 3, 48, 57), + PrefixRange("abc", 3, 48, 57), NoOptimization, // starts with PrefixRange not supported PrefixRange("火花急流", 1, 19968, 40869), NoOptimization, // starts with PrefixRange not supported From f56fe2ccaa14340e33e9261aae18d947a4261d26 Mon Sep 17 00:00:00 2001 From: xieshuaihu Date: Mon, 1 Jul 2024 21:32:39 +0800 Subject: [PATCH 54/79] Fix match error in RapidsShuffleIterator.scala [scala2.13] (#11115) Signed-off-by: xieshuaihu --- .../nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala index 29f36d6f2f7..72c1f935eed 100644 --- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala +++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala @@ -173,7 +173,7 @@ class RapidsShuffleIterator( val (local, remote) = blocksByAddress.partition(ba => ba._1.host == localHost) (local ++ remote).foreach { - case (blockManagerId: BlockManagerId, blockIds: Seq[(BlockId, Long, Int)]) => { + case (blockManagerId: BlockManagerId, blockIds: collection.Seq[(BlockId, Long, Int)]) => { val shuffleRequestsMapIndex: Seq[BlockIdMapIndex] = blockIds.map { case (blockId, _, mapIndex) => /** @@ -193,7 +193,7 @@ class RapidsShuffleIterator( throw new IllegalArgumentException( s"${blockId.getClass} $blockId is not currently supported") } - } + }.toSeq val client = try { transport.makeClient(blockManagerId) From 850365ceac86d40531fed74f45221a81db7d9132 Mon Sep 17 00:00:00 2001 From: MithunR Date: Mon, 1 Jul 2024 13:58:47 -0700 Subject: [PATCH 55/79] Spark 4: Handle ANSI mode in sort_test.py (#11099) * Spark 4: Handle ANSI mode in sort_test.py Fixes #11027. With ANSI mode enabled (like the default in Spark 4), one sees that some tests in `sort_test.py` fail, because they expect ANSI mode to be off. This commit disables running those tests with ANSI enabled, and add a separate test for ANSI on/off. Signed-off-by: MithunR * Refactored not to use disable_ansi_mode. These tests need not be revisited. They test all combinations of ANSI mode, including overflow failures. Signed-off-by: MithunR --------- Signed-off-by: MithunR --- .../src/main/python/sort_test.py | 66 +++++++++++++++---- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/integration_tests/src/main/python/sort_test.py b/integration_tests/src/main/python/sort_test.py index cb905c9fb77..3fe406d180a 100644 --- a/integration_tests/src/main/python/sort_test.py +++ b/integration_tests/src/main/python/sort_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ import pytest -from asserts import assert_gpu_and_cpu_are_equal_collect, assert_gpu_fallback_collect +from asserts import assert_gpu_and_cpu_are_equal_collect, assert_gpu_and_cpu_error, assert_gpu_fallback_collect from conftest import is_not_utc from data_gen import * from marks import allow_non_gpu @@ -224,29 +224,67 @@ def test_multi_orderby_with_limit_single_part(data_gen): assert_gpu_and_cpu_are_equal_collect( lambda spark : binary_op_df(spark, data_gen).coalesce(1).orderBy(f.col('a'), f.col('b').desc()).limit(100)) + # We are not trying all possibilities, just doing a few with numbers so the query works. -@pytest.mark.parametrize('data_gen', [byte_gen, long_gen, float_gen], ids=idfn) -def test_orderby_with_processing(data_gen): +@pytest.mark.parametrize('is_ansi_enabled', [False, True]) +@pytest.mark.parametrize('data_gen', [ByteGen, LongGen, FloatGen], ids=idfn) +def test_orderby_with_processing(data_gen, is_ansi_enabled): + """ + Tests the cases where arithmetic overflows don't occur, in ANSI mode. + Overflow exceptions are tested in test_orderby_with_ansi_overflow_exceptions. + """ + conf = {'spark.sql.ansi.enabled': is_ansi_enabled} + gen = data_gen(min_val=0) if (is_ansi_enabled and data_gen != FloatGen) else data_gen() assert_gpu_and_cpu_are_equal_collect( - # avoid ambiguity in the order by statement for floating point by including a as a backup ordering column - lambda spark : unary_op_df(spark, data_gen).orderBy(f.lit(100) - f.col('a'), f.col('a'))) + # avoid ambiguity in the order by statement for floating point by including `a` as a backup ordering column + lambda spark: unary_op_df(spark, gen).orderBy(f.lit(100) - f.col('a'), f.col('a')), + conf=conf) + + +@pytest.mark.parametrize('data_gen', [long_gen], ids=idfn) +def test_orderby_with_ansi_overflow_exceptions(data_gen): + """ + Test to check that ANSI mode is honoured when there's an order-by with a subtraction expression. + With ANSI mode enabled, the subtraction will overflow, causing an ArithmeticException. + """ + def test_function(spark): + return unary_op_df(spark, data_gen).orderBy(f.lit(100) - f.col('a'), f.col('a')) + + assert_gpu_and_cpu_error(lambda spark: test_function(spark).collect(), + conf=ansi_enabled_conf, + error_message='ArithmeticException') + # We are not trying all possibilities, just doing a few with numbers so the query works. -@pytest.mark.parametrize('data_gen', [byte_gen, long_gen, float_gen], ids=idfn) -def test_orderby_with_processing_and_limit(data_gen): +@pytest.mark.parametrize('is_ansi_enabled', [False, True]) +@pytest.mark.parametrize('data_gen', [ByteGen, LongGen, FloatGen], ids=idfn) +def test_orderby_with_processing_and_limit(data_gen, is_ansi_enabled): + """ + Tests the cases where arithmetic overflows don't occur, in ANSI mode. + Overflow exceptions are tested in test_orderby_with_ansi_overflow_exceptions. + """ + conf = {'spark.sql.ansi.enabled': is_ansi_enabled} + gen = data_gen(min_val=0) if (is_ansi_enabled and data_gen != FloatGen) else data_gen() assert_gpu_and_cpu_are_equal_collect( - # avoid ambiguity in the order by statement for floating point by including a as a backup ordering column - lambda spark : unary_op_df(spark, data_gen).orderBy(f.lit(100) - f.col('a'), f.col('a')).limit(100)) + # avoid ambiguity in the order by statement for floating point by including a as a backup ordering column + lambda spark: unary_op_df(spark, gen).orderBy(f.lit(100) - f.col('a'), f.col('a')).limit(100), conf=conf) # We are not trying all possibilities, just doing a few with numbers so the query works. +@pytest.mark.parametrize('is_ansi_enabled', [False, True]) @pytest.mark.parametrize('data_gen', [StructGen([('child0', long_gen)])], ids=idfn) -def test_single_nested_orderby_with_processing_and_limit(data_gen): +def test_single_nested_orderby_with_processing_and_limit(data_gen, is_ansi_enabled): + """ + Tests the cases where arithmetic overflows don't occur, in ANSI mode. + Overflow exceptions are tested in test_orderby_with_ansi_overflow_exceptions. + """ + conf = {'spark.sql.ansi.enabled': is_ansi_enabled} + data_gen = StructGen([('child0', LongGen(min_val=0) if is_ansi_enabled else LongGen())]) assert_gpu_and_cpu_are_equal_collect( # avoid ambiguity in the order by statement for floating point by including a as a backup ordering column - lambda spark : unary_op_df(spark, data_gen)\ - .orderBy(f.struct(f.lit(100) - f.col('a.child0')), f.col('a'))\ - .limit(100)) + lambda spark: unary_op_df(spark, data_gen)\ + .orderBy(f.struct(f.lit(100) - f.col('a.child0')), f.col('a')).limit(100), + conf=conf) # We are not trying all possibilities, just doing a few with numbers so the query works. @pytest.mark.parametrize('data_gen', [byte_gen, long_gen, float_gen], ids=idfn) From 9bb295a21ce7645aa51551ca83716cd33dce722e Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Tue, 2 Jul 2024 09:40:29 +0800 Subject: [PATCH 56/79] Introduce LORE framework. (#11084) * Introduce lore id * Introduce lore id * Fix type * Fix type * Conf * style * part * Dump * Introduce lore framework * Add tests. * Rename test case Signed-off-by: liurenjie1024 * Fix AQE test * Fix style * Use args to display lore info. * Fix build break * Fix path in loreinfo * Remove path * Fix comments * Update configs * Fix comments * Fix config --------- Signed-off-by: liurenjie1024 --- .../advanced_configs.md | 2 + docs/dev/lore.md | 70 +++++ .../com/nvidia/spark/rapids/DumpUtils.scala | 28 +- .../spark/rapids/GpuAggregateExec.scala | 9 +- .../com/nvidia/spark/rapids/GpuExec.scala | 30 +- .../nvidia/spark/rapids/GpuOverrides.scala | 8 +- .../spark/rapids/GpuTransitionOverrides.scala | 5 + .../com/nvidia/spark/rapids/RapidsConf.scala | 31 ++ .../nvidia/spark/rapids/lore/GpuLore.scala | 295 ++++++++++++++++++ .../spark/rapids/lore/OutputLoreId.scala | 75 +++++ .../com/nvidia/spark/rapids/lore/dump.scala | 106 +++++++ .../nvidia/spark/rapids/lore/package.scala | 35 +++ .../com/nvidia/spark/rapids/lore/replay.scala | 102 ++++++ .../execution/GpuBroadcastExchangeExec.scala | 18 +- .../execution/datasources/GpuWriteFiles.scala | 2 +- .../spark/rapids/lore/GpuLoreSuite.scala | 169 ++++++++++ .../spark/rapids/lore/OutputLoreIdSuite.scala | 55 ++++ 17 files changed, 1029 insertions(+), 11 deletions(-) create mode 100644 docs/dev/lore.md create mode 100644 sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/GpuLore.scala create mode 100644 sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/OutputLoreId.scala create mode 100644 sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/dump.scala create mode 100644 sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/package.scala create mode 100644 sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/replay.scala create mode 100644 tests/src/test/scala/com/nvidia/spark/rapids/lore/GpuLoreSuite.scala create mode 100644 tests/src/test/scala/com/nvidia/spark/rapids/lore/OutputLoreIdSuite.scala diff --git a/docs/additional-functionality/advanced_configs.md b/docs/additional-functionality/advanced_configs.md index 033e332b99c..f5d511cbbc5 100644 --- a/docs/additional-functionality/advanced_configs.md +++ b/docs/additional-functionality/advanced_configs.md @@ -136,6 +136,8 @@ Name | Description | Default Value | Applicable at spark.rapids.sql.json.read.decimal.enabled|When reading a quoted string as a decimal Spark supports reading non-ascii unicode digits, and the RAPIDS Accelerator does not.|true|Runtime spark.rapids.sql.json.read.double.enabled|JSON reading is not 100% compatible when reading doubles.|true|Runtime spark.rapids.sql.json.read.float.enabled|JSON reading is not 100% compatible when reading floats.|true|Runtime +spark.rapids.sql.lore.dumpPath|The path to dump the LORE nodes' input data. This must be set if spark.rapids.sql.lore.idsToDump has been set. The data of each LORE node will be dumped to a subfolder with name 'loreId-' under this path. For more details, please refer to [the LORE documentation](../dev/lore.md).|None|Runtime +spark.rapids.sql.lore.idsToDump|Specify the LORE ids of operators to dump. The format is a comma separated list of LORE ids. For example: "1[0]" will dump partition 0 of input of gpu operator with lore id 1. For more details, please refer to [the LORE documentation](../dev/lore.md). If this is not set, no data will be dumped.|None|Runtime spark.rapids.sql.mode|Set the mode for the Rapids Accelerator. The supported modes are explainOnly and executeOnGPU. This config can not be changed at runtime, you must restart the application for it to take affect. The default mode is executeOnGPU, which means the RAPIDS Accelerator plugin convert the Spark operations and execute them on the GPU when possible. The explainOnly mode allows running queries on the CPU and the RAPIDS Accelerator will evaluate the queries as if it was going to run on the GPU. The explanations of what would have run on the GPU and why are output in log messages. When using explainOnly mode, the default explain output is ALL, this can be changed by setting spark.rapids.sql.explain. See that config for more details.|executeongpu|Startup spark.rapids.sql.optimizer.joinReorder.enabled|When enabled, joins may be reordered for improved query performance|true|Runtime spark.rapids.sql.python.gpu.enabled|This is an experimental feature and is likely to change in the future. Enable (true) or disable (false) support for scheduling Python Pandas UDFs with GPU resources. When enabled, pandas UDFs are assumed to share the same GPU that the RAPIDs accelerator uses and will honor the python GPU configs|false|Runtime diff --git a/docs/dev/lore.md b/docs/dev/lore.md new file mode 100644 index 00000000000..d6b28877ae7 --- /dev/null +++ b/docs/dev/lore.md @@ -0,0 +1,70 @@ +--- +layout: page +title: The Local Replay Framework +nav_order: 13 +parent: Developer Overview +--- + +# Local Replay Framework + +## Overview + +LORE (the local replay framework) is a tool that allows developer to replay the execution of a +gpu operator in local environment, so that developer could debug and profile the operator for +performance analysis. In high level it works as follows: + +1. Each gpu operator will be assigned a LORE id, which is a unique identifier for the operator. + This id is guaranteed to be unique within the same query, and guaranteed to be same when two + sql executions have same sql, same configuration, and same data. +2. In the first run of the query, developer could found the LORE id of the operator they are + interested in by checking spark ui, where LORE id usually appears in the arguments of operator. +3. In the second run of the query, developer needs to configure the LORE ids of the operators they + are interested in, and LORE will dump the input data of the operator to given path. +4. Developer could copy the dumped data to local environment, and replay the operator in local + environment. + +## Configuration + +By default, LORE id will always be generated for operators, but user could disable this behavior +by setting `spark.rapids.sql.lore.tag.enabled` to `false`. + +To tell LORE the LORE ids of the operators you are interested in, you need to set +`spark.rapids.sql.lore.idsToDump`. For example, you could set it to "1[*], 2[*], 3[*]" to tell +LORE to dump all partitions of input data of operators with id 1, 2, or 3. You can also only dump +some partition of the operator's input by appending partition numbers to lore ids. For example, +"1[0 4-6 7], 2[*]" tell LORE to dump operator with LORE id 1, but only dump partition 0, 4, 5, 6, +and 7. But for operator with LORE id 2, it will dump all partitions. + +You also need to set `spark.rapids.sql.lore.dumpPath` to tell LORE where to dump the data, the +value of which should point to a directory. All dumped data of a query will live in this +directory. A typical directory hierarchy would look like this: + +```console ++ loreId-10/ + - plan.meta + + input-0/ + - rdd.meta + + partition-0/ + - partition.meta + - batch-0.parquet + - batch-1.parquet + + partition-1/ + - partition.meta + - batch-0.parquet + + input-1/ + - rdd.meta + + partition-0/ + - partition.meta + - batch-0.parquet + - batch-1.parquet + ++ loreId-15/ + - plan.meta + + input-0/ + - rdd.meta + + partition-0/ + - partition.meta + - batch-0.parquet +``` + + diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/DumpUtils.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/DumpUtils.scala index bf949897c78..21d2de6ad68 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/DumpUtils.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/DumpUtils.scala @@ -15,7 +15,7 @@ */ package com.nvidia.spark.rapids -import java.io.{File, FileOutputStream} +import java.io.{File, FileOutputStream, OutputStream} import java.util.Random import scala.collection.mutable @@ -23,7 +23,7 @@ import scala.collection.mutable.ArrayBuffer import ai.rapids.cudf._ import ai.rapids.cudf.ColumnWriterOptions._ -import com.nvidia.spark.rapids.Arm.withResource +import com.nvidia.spark.rapids.Arm.{closeOnExcept, withResource} import org.apache.commons.io.IOUtils import org.apache.hadoop.conf.Configuration @@ -82,6 +82,23 @@ object DumpUtils extends Logging { } } + /** + * Dump columnar batch to output stream in parquet format.
+ * + * @param columnarBatch The columnar batch to be dumped, should be GPU columnar batch. It + * should be closed by caller. + * @param outputStream Will be closed after writing. + */ + def dumpToParquet(columnarBatch: ColumnarBatch, outputStream: OutputStream): Unit = { + closeOnExcept(outputStream) { _ => + withResource(GpuColumnVector.from(columnarBatch)) { table => + withResource(new ParquetDumper(outputStream, table)) { dumper => + dumper.writeTable(table) + } + } + } + } + /** * Debug utility to dump table to parquet file.
* It's running on GPU. Parquet column names are generated from table column type info.
@@ -129,12 +146,15 @@ object DumpUtils extends Logging { } // parquet dumper -class ParquetDumper(path: String, table: Table) extends HostBufferConsumer +class ParquetDumper(private val outputStream: OutputStream, table: Table) extends HostBufferConsumer with AutoCloseable { - private[this] val outputStream = new FileOutputStream(path) private[this] val tempBuffer = new Array[Byte](128 * 1024) private[this] val buffers = mutable.Queue[(HostMemoryBuffer, Long)]() + def this(path: String, table: Table) = { + this(new FileOutputStream(path), table) + } + val tableWriter: TableWriter = { // avoid anything conversion, just dump as it is val builder = ParquetDumper.parquetWriterOptionsFromTable(ParquetWriterOptions.builder(), table) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuAggregateExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuAggregateExec.scala index 7e6a1056d01..b35e687d185 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuAggregateExec.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuAggregateExec.scala @@ -1827,6 +1827,7 @@ case class GpuHashAggregateExec( |${ExplainUtils.generateFieldString("Functions", aggregateExpressions)} |${ExplainUtils.generateFieldString("Aggregate Attributes", aggregateAttributes)} |${ExplainUtils.generateFieldString("Results", resultExpressions)} + |Lore: ${loreArgs.mkString(", ")} |""".stripMargin } @@ -1955,10 +1956,12 @@ case class GpuHashAggregateExec( truncatedString(allAggregateExpressions, "[", ", ", "]", maxFields) val outputString = truncatedString(output, "[", ", ", "]", maxFields) if (verbose) { - s"GpuHashAggregate(keys=$keyString, functions=$functionString, output=$outputString)" + s"$nodeName (keys=$keyString, functions=$functionString, output=$outputString) " + + s"""${loreArgs.mkString(", ")}""" } else { - s"GpuHashAggregate(keys=$keyString, functions=$functionString)," + - s" filters=${aggregateExpressions.map(_.filter)})" + s"$nodeName (keys=$keyString, functions=$functionString)," + + s" filters=${aggregateExpressions.map(_.filter)})" + + s""" ${loreArgs.mkString(", ")}""" } } // diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExec.scala index d83f20113b2..e93ac40b5bd 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExec.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExec.scala @@ -19,7 +19,10 @@ package com.nvidia.spark.rapids import ai.rapids.cudf.NvtxColor import com.nvidia.spark.rapids.Arm.withResource import com.nvidia.spark.rapids.filecache.FileCacheConf +import com.nvidia.spark.rapids.lore.{GpuLore, GpuLoreDumpRDD} +import com.nvidia.spark.rapids.lore.GpuLore.{loreIdOf, LORE_DUMP_PATH_TAG, LORE_DUMP_RDD_TAG} import com.nvidia.spark.rapids.shims.SparkShimImpl +import org.apache.hadoop.fs.Path import org.apache.spark.internal.Logging import org.apache.spark.rapids.LocationPreservingMapPartitionsRDD @@ -385,7 +388,8 @@ trait GpuExec extends SparkPlan { this.getTagValue(GpuExec.TASK_METRICS_TAG) final override def doExecuteColumnar(): RDD[ColumnarBatch] = { - val orig = internalDoExecuteColumnar() + this.dumpLoreMetaInfo() + val orig = this.dumpLoreRDD(internalDoExecuteColumnar()) val metrics = getTaskMetrics metrics.map { gpuMetrics => // This is ugly, but it reduces the need to change all exec nodes, so we are doing it here @@ -396,5 +400,29 @@ trait GpuExec extends SparkPlan { }.getOrElse(orig) } + override def stringArgs: Iterator[Any] = super.stringArgs ++ loreArgs + + protected def loreArgs: Iterator[String] = { + val loreIdStr = loreIdOf(this).map(id => s"[loreId=$id]") + val lorePathStr = getTagValue(LORE_DUMP_PATH_TAG).map(path => s"[lorePath=$path]") + val loreRDDInfoStr = getTagValue(LORE_DUMP_RDD_TAG).map(info => s"[loreRDDInfo=$info]") + + List(loreIdStr, lorePathStr, loreRDDInfoStr).flatten.iterator + } + + private def dumpLoreMetaInfo(): Unit = { + getTagValue(LORE_DUMP_PATH_TAG).foreach { rootPath => + GpuLore.dumpPlan(this, new Path(rootPath)) + } + } + + protected def dumpLoreRDD(inner: RDD[ColumnarBatch]): RDD[ColumnarBatch] = { + getTagValue(LORE_DUMP_RDD_TAG).map { info => + val rdd = new GpuLoreDumpRDD(info, inner) + rdd.saveMeta() + rdd + }.getOrElse(inner) + } + protected def internalDoExecuteColumnar(): RDD[ColumnarBatch] } diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 9e26cf751f4..73475ef36f5 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -25,6 +25,7 @@ import scala.util.control.NonFatal import ai.rapids.cudf.DType import com.nvidia.spark.rapids.RapidsConf.{SUPPRESS_PLANNING_FAILURE, TEST_CONF} import com.nvidia.spark.rapids.jni.GpuTimeZoneDB +import com.nvidia.spark.rapids.lore.GpuLore import com.nvidia.spark.rapids.shims._ import com.nvidia.spark.rapids.window.{GpuDenseRank, GpuLag, GpuLead, GpuPercentRank, GpuRank, GpuRowNumber, GpuSpecialFrameBoundary, GpuWindowExecMeta, GpuWindowSpecDefinitionMeta} import org.apache.hadoop.fs.Path @@ -4708,7 +4709,12 @@ case class GpuOverrides() extends Rule[SparkPlan] with Logging { } } } - GpuOverrides.doConvertPlan(wrap, conf, optimizations) + val convertedPlan = GpuOverrides.doConvertPlan(wrap, conf, optimizations) + if (conf.isTagLoreIdEnabled) { + GpuLore.tagForLore(convertedPlan, conf) + } else { + convertedPlan + } } } } diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuTransitionOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuTransitionOverrides.scala index 48f9de5a61a..c8596f983d9 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuTransitionOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuTransitionOverrides.scala @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicInteger import scala.annotation.tailrec import scala.collection.mutable +import com.nvidia.spark.rapids.lore.GpuLore import com.nvidia.spark.rapids.shims.{GpuBatchScanExec, SparkShimImpl} import org.apache.spark.SparkContext @@ -823,6 +824,10 @@ class GpuTransitionOverrides extends Rule[SparkPlan] { updatedPlan = fixupAdaptiveExchangeReuse(updatedPlan) } + if (rapidsConf.isTagLoreIdEnabled) { + updatedPlan = GpuLore.tagForLore(updatedPlan, rapidsConf) + } + if (rapidsConf.logQueryTransformations) { logWarning(s"Transformed query:" + s"\nOriginal Plan:\n$plan\nTransformed Plan:\n$updatedPlan") diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index aad4f05b334..406d09a7a32 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -23,6 +23,7 @@ import scala.collection.mutable.{HashMap, ListBuffer} import ai.rapids.cudf.Cuda import com.nvidia.spark.rapids.jni.RmmSpark.OomInjectionType +import com.nvidia.spark.rapids.lore.{LoreId, OutputLoreId} import org.apache.spark.SparkConf import org.apache.spark.internal.Logging @@ -2308,6 +2309,28 @@ val SHUFFLE_COMPRESSION_LZ4_CHUNK_SIZE = conf("spark.rapids.shuffle.compression. .booleanConf .createWithDefault(false) + val TAG_LORE_ID_ENABLED = conf("spark.rapids.sql.lore.tag.enabled") + .doc("Enable add a LORE id to each gpu plan node") + .internal() + .booleanConf + .createWithDefault(true) + + val LORE_DUMP_IDS = conf("spark.rapids.sql.lore.idsToDump") + .doc("Specify the LORE ids of operators to dump. The format is a comma separated list of " + + "LORE ids. For example: \"1[0]\" will dump partition 0 of input of gpu operator " + + "with lore id 1. For more details, please refer to " + + "[the LORE documentation](../dev/lore.md). If this is not set, no data will be dumped.") + .stringConf + .createOptional + + val LORE_DUMP_PATH = conf("spark.rapids.sql.lore.dumpPath") + .doc(s"The path to dump the LORE nodes' input data. This must be set if ${LORE_DUMP_IDS.key} " + + "has been set. The data of each LORE node will be dumped to a subfolder with name " + + "'loreId-' under this path. For more details, please refer to " + + "[the LORE documentation](../dev/lore.md).") + .stringConf + .createOptional + private def printSectionHeader(category: String): Unit = println(s"\n### $category") @@ -3121,6 +3144,14 @@ class RapidsConf(conf: Map[String, String]) extends Logging { lazy val isDeltaLowShuffleMergeEnabled: Boolean = get(ENABLE_DELTA_LOW_SHUFFLE_MERGE) + lazy val isTagLoreIdEnabled: Boolean = get(TAG_LORE_ID_ENABLED) + + lazy val loreDumpIds: Map[LoreId, OutputLoreId] = get(LORE_DUMP_IDS) + .map(OutputLoreId.parse) + .getOrElse(Map.empty) + + lazy val loreDumpPath: Option[String] = get(LORE_DUMP_PATH) + private val optimizerDefaults = Map( // this is not accurate because CPU projections do have a cost due to appending values // to each row that is produced, but this needs to be a really small number because diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/GpuLore.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/GpuLore.scala new file mode 100644 index 00000000000..a51a1e13a5e --- /dev/null +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/GpuLore.scala @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.lore + +import java.util.concurrent.{ConcurrentHashMap, ConcurrentMap} +import java.util.concurrent.atomic.AtomicInteger + +import scala.collection.mutable +import scala.reflect.ClassTag + +import com.nvidia.spark.rapids.{GpuColumnarToRowExec, GpuExec, RapidsConf} +import com.nvidia.spark.rapids.Arm.withResource +import com.nvidia.spark.rapids.shims.SparkShimImpl +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path + +import org.apache.spark.SparkEnv +import org.apache.spark.broadcast.Broadcast +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.trees.TreeNodeTag +import org.apache.spark.sql.execution.{BaseSubqueryExec, ExecSubqueryExpression, ReusedSubqueryExec, SparkPlan, SQLExecution} +import org.apache.spark.sql.execution.adaptive.BroadcastQueryStageExec +import org.apache.spark.sql.rapids.execution.{GpuBroadcastExchangeExec, GpuCustomShuffleReaderExec} +import org.apache.spark.sql.types.DataType +import org.apache.spark.util.SerializableConfiguration + +case class LoreRDDMeta(numPartitions: Int, outputPartitions: Seq[Int], attrs: Seq[Attribute]) + +case class LoreRDDPartitionMeta(numBatches: Int, dataType: Seq[DataType]) + +trait GpuLoreRDD { + def rootPath: Path + + def pathOfMeta: Path = new Path(rootPath, "rdd.meta") + + def pathOfPartition(partitionIndex: Int): Path = { + new Path(rootPath, s"partition-$partitionIndex") + } + + def pathOfPartitionMeta(partitionIndex: Int): Path = { + new Path(pathOfPartition(partitionIndex), "partition.meta") + } + + def pathOfBatch(partitionIndex: Int, batchIndex: Int): Path = { + new Path(pathOfPartition(partitionIndex), s"batch-$batchIndex.parquet") + } +} + + +object GpuLore { + /** + * Lore id of a plan node. + */ + val LORE_ID_TAG: TreeNodeTag[String] = new TreeNodeTag[String]("rapids.gpu.lore.id") + /** + * When a [[GpuExec]] node has this tag, it means that this node is a root node whose meta and + * input should be dumped. + */ + val LORE_DUMP_PATH_TAG: TreeNodeTag[String] = new TreeNodeTag[String]("rapids.gpu.lore.dump.path") + /** + * When a [[GpuExec]] node has this tag, it means that this node is a child node whose data + * should be dumped. + */ + val LORE_DUMP_RDD_TAG: TreeNodeTag[LoreDumpRDDInfo] = new TreeNodeTag[LoreDumpRDDInfo]( + "rapids.gpu.lore.dump.rdd.info") + + def pathOfRootPlanMeta(rootPath: Path): Path = { + new Path(rootPath, "plan.meta") + } + + def dumpPlan[T <: SparkPlan : ClassTag](plan: T, rootPath: Path): Unit = { + dumpObject(plan, pathOfRootPlanMeta(rootPath), + SparkShimImpl.sessionFromPlan(plan).sparkContext.hadoopConfiguration) + } + + def dumpObject[T: ClassTag](obj: T, path: Path, hadoopConf: Configuration): Unit = { + withResource(path.getFileSystem(hadoopConf)) { fs => + withResource(fs.create(path, false)) { fout => + val serializerStream = SparkEnv.get.serializer.newInstance().serializeStream(fout) + withResource(serializerStream) { ser => + ser.writeObject(obj) + } + } + } + } + + def loadObject[T: ClassTag](path: Path, hadoopConf: Configuration): T = { + withResource(path.getFileSystem(hadoopConf)) { fs => + withResource(fs.open(path)) { fin => + val serializerStream = SparkEnv.get.serializer.newInstance().deserializeStream(fin) + withResource(serializerStream) { ser => + ser.readObject().asInstanceOf[T] + } + } + } + } + + def pathOfChild(rootPath: Path, childIndex: Int): Path = { + new Path(rootPath, s"input-$childIndex") + } + + def restoreGpuExec(rootPath: Path, spark: SparkSession): GpuExec = { + val rootExec = loadObject[GpuExec](pathOfRootPlanMeta(rootPath), + spark.sparkContext.hadoopConfiguration) + + checkUnsupportedOperator(rootExec) + + val broadcastHadoopConf = { + val sc = spark.sparkContext + sc.broadcast(new SerializableConfiguration(spark.sparkContext.hadoopConfiguration)) + } + + // Load children + val newChildren = rootExec.children.zipWithIndex.map { case (plan, idx) => + val newChild = GpuLoreReplayExec(idx, rootPath.toString, broadcastHadoopConf) + plan match { + case b: GpuBroadcastExchangeExec => + b.withNewChildren(Seq(newChild)) + case b: BroadcastQueryStageExec => + b.broadcast.withNewChildren(Seq(newChild)) + case _ => newChild + } + } + + var nextId = rootExec.children.length + + rootExec.transformExpressionsUp { + case sub: ExecSubqueryExpression => + val newSub = restoreSubqueryPlan(nextId, sub, rootPath, broadcastHadoopConf) + nextId += 1 + newSub + }.withNewChildren(newChildren).asInstanceOf[GpuExec] + } + + private def restoreSubqueryPlan(id: Int, sub: ExecSubqueryExpression, + rootPath: Path, hadoopConf: Broadcast[SerializableConfiguration]): ExecSubqueryExpression = { + val innerPlan = sub.plan.child + + if (innerPlan.isInstanceOf[GpuExec]) { + var newChild: SparkPlan = GpuLoreReplayExec(id, rootPath.toString, hadoopConf) + + if (!innerPlan.supportsColumnar) { + newChild = GpuColumnarToRowExec(newChild) + } + val newSubqueryExec = sub.plan match { + case ReusedSubqueryExec(subqueryExec) => subqueryExec.withNewChildren(Seq(newChild)) + .asInstanceOf[BaseSubqueryExec] + case p: BaseSubqueryExec => p.withNewChildren(Seq(newChild)) + .asInstanceOf[BaseSubqueryExec] + } + sub.withNewPlan(newSubqueryExec) + } else { + throw new IllegalArgumentException(s"Subquery plan ${innerPlan.getClass.getSimpleName} " + + s"is not a GpuExec") + } + } + + /** + * Lore id generator. Key is [[SQLExecution.EXECUTION_ID_KEY]]. + */ + private val idGen: ConcurrentMap[String, AtomicInteger] = + new ConcurrentHashMap[String, AtomicInteger]() + + private def nextLoreIdOf(plan: SparkPlan): Option[Int] = { + // When the execution id is not set, it means there is no actual execution happening, in this + // case we don't need to generate lore id. + Option(SparkShimImpl.sessionFromPlan(plan) + .sparkContext + .getLocalProperty(SQLExecution.EXECUTION_ID_KEY)) + .map { executionId => + idGen.computeIfAbsent(executionId, _ => new AtomicInteger(0)).getAndIncrement() + } + } + + def tagForLore(sparkPlan: SparkPlan, rapidsConf: RapidsConf): SparkPlan = { + val loreDumpIds = rapidsConf.loreDumpIds + + val newPlan = if (loreDumpIds.nonEmpty) { + // We need to dump the output of nodes with the lore id in the dump ids + val loreOutputRootPath = rapidsConf.loreDumpPath.getOrElse(throw + new IllegalArgumentException(s"${RapidsConf.LORE_DUMP_PATH.key} must be set " + + s"when ${RapidsConf.LORE_DUMP_IDS.key} is set.")) + + val spark = SparkShimImpl.sessionFromPlan(sparkPlan) + val hadoopConf = { + val sc = spark.sparkContext + sc.broadcast(new SerializableConfiguration(sc.hadoopConfiguration)) + } + + val subqueries = mutable.Set.empty[SparkPlan] + + sparkPlan.foreachUp { + case g: GpuExec => + nextLoreIdOf(g).foreach { loreId => + g.setTagValue(LORE_ID_TAG, loreId.toString) + + loreDumpIds.get(loreId).foreach { outputLoreIds => + checkUnsupportedOperator(g) + val currentExecRootPath = new Path(loreOutputRootPath, s"loreId-$loreId") + g.setTagValue(LORE_DUMP_PATH_TAG, currentExecRootPath.toString) + val loreOutputInfo = LoreOutputInfo(outputLoreIds, + currentExecRootPath.toString) + + g.children.zipWithIndex.foreach { + case (child, idx) => + val dumpRDDInfo = LoreDumpRDDInfo(idx, loreOutputInfo, child.output, hadoopConf) + child match { + case c: BroadcastQueryStageExec => + c.broadcast.setTagValue(LORE_DUMP_RDD_TAG, dumpRDDInfo) + case o => o.setTagValue(LORE_DUMP_RDD_TAG, dumpRDDInfo) + } + } + + var nextId = g.children.length + g.transformExpressionsUp { + case sub: ExecSubqueryExpression => + if (spark.sessionState.conf.subqueryReuseEnabled) { + if (!subqueries.contains(sub.plan.canonicalized)) { + subqueries += sub.plan.canonicalized + } else { + throw new IllegalArgumentException("Subquery reuse is enabled, and we found" + + " duplicated subqueries, which is currently not supported by LORE.") + } + } + tagSubqueryPlan(nextId, sub, loreOutputInfo, hadoopConf) + nextId += 1 + sub + } + } + } + case _ => + } + + sparkPlan + + } else { + // We don't need to dump the output of the nodes, just tag the lore id + sparkPlan.foreachUp { + case g: GpuExec => + nextLoreIdOf(g).foreach { loreId => + g.setTagValue(LORE_ID_TAG, loreId.toString) + } + case _ => + } + + sparkPlan + } + + newPlan + } + + def loreIdOf(node: SparkPlan): Option[String] = { + node.getTagValue(LORE_ID_TAG) + } + + private def tagSubqueryPlan(id: Int, sub: ExecSubqueryExpression, + loreOutputInfo: LoreOutputInfo, hadoopConf: Broadcast[SerializableConfiguration]) = { + val innerPlan = sub.plan.child + if (innerPlan.isInstanceOf[GpuExec]) { + val dumpRDDInfo = LoreDumpRDDInfo(id, loreOutputInfo, innerPlan.output, + hadoopConf) + innerPlan match { + case p: GpuColumnarToRowExec => p.child.setTagValue(LORE_DUMP_RDD_TAG, dumpRDDInfo) + case c => c.setTagValue(LORE_DUMP_RDD_TAG, dumpRDDInfo) + } + } else { + throw new IllegalArgumentException(s"Subquery plan ${innerPlan.getClass.getSimpleName} " + + s"is not a GpuExec") + } + } + + private def checkUnsupportedOperator(plan: SparkPlan): Unit = { + if (plan.children.isEmpty || + plan.isInstanceOf[GpuCustomShuffleReaderExec] + ) { + throw new UnsupportedOperationException(s"Currently we don't support dumping input of " + + s"${plan.getClass.getSimpleName} operator.") + } + } +} diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/OutputLoreId.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/OutputLoreId.scala new file mode 100644 index 00000000000..28fa0b2dbbf --- /dev/null +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/OutputLoreId.scala @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.lore + +import org.apache.hadoop.fs.Path + +case class OutputLoreId(loreId: LoreId, partitionIds: Set[Int]) { + def outputAllParitions: Boolean = partitionIds.isEmpty + + def shouldOutputPartition(partitionId: Int): Boolean = outputAllParitions || + partitionIds.contains(partitionId) +} + +case class LoreOutputInfo(outputLoreId: OutputLoreId, pathStr: String) { + def path: Path = new Path(pathStr) +} + +object OutputLoreId { + private val PARTITION_ID_RANGE_REGEX = raw"(\d+)-(\d+)".r("start", "end") + private val PARTITION_ID_REGEX = raw"(\d+)".r("partitionId") + private val PARTITION_ID_SEP_REGEX = raw" +".r + + private val OUTPUT_LORE_ID_SEP_REGEX = ", *".r + private val OUTPUT_LORE_ID_REGEX = + raw"(?\d+)(\[(?.*)\])?".r + + def apply(loreId: Int): OutputLoreId = OutputLoreId(loreId, Set.empty) + + def apply(inputStr: String): OutputLoreId = { + OUTPUT_LORE_ID_REGEX.findFirstMatchIn(inputStr).map { m => + val loreId = m.group("loreId").toInt + val partitionIds: Set[Int] = m.group("partitionIds") match { + case partitionIdsStr if partitionIdsStr != null => + PARTITION_ID_SEP_REGEX.split(partitionIdsStr).flatMap { + case PARTITION_ID_REGEX(partitionId) => + Seq(partitionId.toInt) + case PARTITION_ID_RANGE_REGEX(start, end) => + start.toInt until end.toInt + case "*" => Set.empty + case partitionIdStr => throw new IllegalArgumentException(s"Invalid partition " + + s"id: $partitionIdStr") + }.toSet + case null => { + throw new IllegalArgumentException(s"Invalid output lore id string: $inputStr, " + + s"partition ids not found!") + } + } + OutputLoreId(loreId, partitionIds) + }.getOrElse(throw new IllegalArgumentException(s"Invalid output lore ids: $inputStr")) + } + + def parse(inputStr: String): OutputLoreIds = { + require(inputStr != null, "inputStr should not be null") + + OUTPUT_LORE_ID_SEP_REGEX.split(inputStr).map(OutputLoreId(_)).map { outputLoreId => + outputLoreId.loreId -> outputLoreId + }.toMap + } +} + + diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/dump.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/dump.scala new file mode 100644 index 00000000000..1b9967e1bf4 --- /dev/null +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/dump.scala @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.lore + +import com.nvidia.spark.rapids.{DumpUtils, GpuColumnVector} +import com.nvidia.spark.rapids.GpuCoalesceExec.EmptyPartition +import com.nvidia.spark.rapids.lore.GpuLore.pathOfChild +import org.apache.hadoop.fs.Path + +import org.apache.spark.{Partition, SparkContext, TaskContext} +import org.apache.spark.broadcast.Broadcast +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.rapids.execution.GpuBroadcastHelper +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.vectorized.ColumnarBatch +import org.apache.spark.util.SerializableConfiguration + + +case class LoreDumpRDDInfo(idxInParent: Int, loreOutputInfo: LoreOutputInfo, attrs: Seq[Attribute], + hadoopConf: Broadcast[SerializableConfiguration]) + +class GpuLoreDumpRDD(info: LoreDumpRDDInfo, input: RDD[ColumnarBatch]) + extends RDD[ColumnarBatch](input) with GpuLoreRDD { + override def rootPath: Path = pathOfChild(info.loreOutputInfo.path, info.idxInParent) + + def saveMeta(): Unit = { + val meta = LoreRDDMeta(input.getNumPartitions, this.getPartitions.map(_.index), info.attrs) + GpuLore.dumpObject(meta, pathOfMeta, this.context.hadoopConfiguration) + } + + override def compute(split: Partition, context: TaskContext): Iterator[ColumnarBatch] = { + if (info.loreOutputInfo.outputLoreId.shouldOutputPartition(split.index)) { + val originalIter = input.compute(split, context) + new Iterator[ColumnarBatch] { + var batchIdx: Int = -1 + var nextBatch: Option[ColumnarBatch] = None + + override def hasNext: Boolean = { + if (batchIdx == -1) { + loadNextBatch() + } + nextBatch.isDefined + } + + override def next(): ColumnarBatch = { + val ret = dumpCurrentBatch() + loadNextBatch() + if (!hasNext) { + // This is the last batch, save the partition meta + val partitionMeta = LoreRDDPartitionMeta(batchIdx, GpuColumnVector.extractTypes(ret)) + GpuLore.dumpObject(partitionMeta, pathOfPartitionMeta(split.index), + info.hadoopConf.value.value) + } + ret + } + + private def dumpCurrentBatch(): ColumnarBatch = { + val outputPath = pathOfBatch(split.index, batchIdx) + val outputStream = outputPath.getFileSystem(info.hadoopConf.value.value) + .create(outputPath, false) + DumpUtils.dumpToParquet(nextBatch.get, outputStream) + nextBatch.get + } + + private def loadNextBatch(): Unit = { + if (originalIter.hasNext) { + nextBatch = Some(originalIter.next()) + } else { + nextBatch = None + } + batchIdx += 1 + } + } + } else { + input.compute(split, context) + } + } + + override protected def getPartitions: Array[Partition] = { + input.partitions + } +} + +class SimpleRDD(_sc: SparkContext, data: Broadcast[Any], schema: StructType) extends + RDD[ColumnarBatch](_sc, Nil) { + override def compute(split: Partition, context: TaskContext): Iterator[ColumnarBatch] = { + Seq(GpuBroadcastHelper.getBroadcastBatch(data, schema)).iterator + } + + override protected def getPartitions: Array[Partition] = Array(EmptyPartition(0)) +} diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/package.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/package.scala new file mode 100644 index 00000000000..f304ea07d97 --- /dev/null +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/package.scala @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids + +/** + * Lore framework is used for dumping input data of a gpu executor to disk so that it can be + * replayed in local environment for performance analysis. + *
+ * When [[RapidsConf.TAG_LORE_ID_ENABLED]] is set, during the planning phase we will tag a lore + * id to each gpu operator. Lore id is guaranteed to be unique within a query, and it's supposed + * to be same for operators with same plan. + *
+ * When [[RapidsConf.LORE_DUMP_IDS]] is set, during the execution phase we will dump the input + * data of gpu operators with lore id to disk. The dumped data can be replayed in local + * environment. The dumped data will reside in [[RapidsConf.LORE_DUMP_PATH]]. For more details, + * please refer to `docs/dev/lore.md`. + */ +package object lore { + type LoreId = Int + type OutputLoreIds = Map[LoreId, OutputLoreId] +} diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/replay.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/replay.scala new file mode 100644 index 00000000000..ffbe207646a --- /dev/null +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/lore/replay.scala @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.lore + +import ai.rapids.cudf.Table +import com.nvidia.spark.rapids.{GpuColumnVector, GpuExec} +import com.nvidia.spark.rapids.Arm.withResource +import org.apache.commons.io.IOUtils +import org.apache.hadoop.fs.Path + +import org.apache.spark.{Partition, SparkContext, TaskContext} +import org.apache.spark.broadcast.Broadcast +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.execution.LeafExecNode +import org.apache.spark.sql.vectorized.ColumnarBatch +import org.apache.spark.util.SerializableConfiguration + +case class GpuLoreReplayExec(idxInParent: Int, parentRootPath: String, + hadoopConf: Broadcast[SerializableConfiguration]) + extends LeafExecNode + with GpuExec { + private lazy val rdd = new GpuLoreReplayRDD(sparkSession.sparkContext, + GpuLore.pathOfChild(new Path(parentRootPath), idxInParent).toString, hadoopConf) + override def output: Seq[Attribute] = rdd.loreRDDMeta.attrs + + override def doExecute(): RDD[InternalRow] = { + throw new UnsupportedOperationException("LoreReplayExec does not support row mode") + } + + override protected def internalDoExecuteColumnar(): RDD[ColumnarBatch] = { + rdd + } +} + +class GpuLoreReplayRDD(sc: SparkContext, rootPathStr: String, + hadoopConf: Broadcast[SerializableConfiguration]) + extends RDD[ColumnarBatch](sc, Nil) with GpuLoreRDD { + + override def rootPath: Path = new Path(rootPathStr) + + private[lore] val loreRDDMeta: LoreRDDMeta = GpuLore.loadObject(pathOfMeta, sc + .hadoopConfiguration) + + override def compute(split: Partition, context: TaskContext): Iterator[ColumnarBatch] = { + val partitionPath = pathOfPartition(split.index) + withResource(partitionPath.getFileSystem(hadoopConf.value.value)) { fs => + if (!fs.exists(partitionPath)) { + Iterator.empty + } else { + val partitionMeta = GpuLore.loadObject[LoreRDDPartitionMeta]( + pathOfPartitionMeta(split.index), hadoopConf.value.value) + new Iterator[ColumnarBatch] { + private var batchIdx: Int = 0 + + override def hasNext: Boolean = { + batchIdx < partitionMeta.numBatches + } + + override def next(): ColumnarBatch = { + val batchPath = pathOfBatch(split.index, batchIdx) + val ret = withResource(batchPath.getFileSystem(hadoopConf.value.value)) { fs => + if (!fs.exists(batchPath)) { + throw new IllegalStateException(s"Batch file $batchPath does not exist") + } + withResource(fs.open(batchPath)) { fin => + val buffer = IOUtils.toByteArray(fin) + withResource(Table.readParquet(buffer)) { restoredTable => + GpuColumnVector.from(restoredTable, partitionMeta.dataType.toArray) + } + } + + } + batchIdx += 1 + ret + } + } + } + } + } + + override protected def getPartitions: Array[Partition] = { + (0 until loreRDDMeta.numPartitions).map(LoreReplayPartition).toArray + } +} + +case class LoreReplayPartition(override val index: Int) extends Partition diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastExchangeExec.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastExchangeExec.scala index 51c6f52d97e..bd30459d63e 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastExchangeExec.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastExchangeExec.scala @@ -31,6 +31,8 @@ import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.Arm.{closeOnExcept, withResource} import com.nvidia.spark.rapids.GpuMetric._ import com.nvidia.spark.rapids.RapidsPluginImplicits._ +import com.nvidia.spark.rapids.lore.{GpuLoreDumpRDD, SimpleRDD} +import com.nvidia.spark.rapids.lore.GpuLore.LORE_DUMP_RDD_TAG import com.nvidia.spark.rapids.shims.{ShimBroadcastExchangeLike, ShimUnaryExecNode, SparkShimImpl} import org.apache.spark.SparkException @@ -486,7 +488,9 @@ abstract class GpuBroadcastExchangeExecBase( throw new IllegalStateException("A canonicalized plan is not supposed to be executed.") } try { - relationFuture.get(timeout, TimeUnit.SECONDS).asInstanceOf[Broadcast[T]] + val ret = relationFuture.get(timeout, TimeUnit.SECONDS) + doLoreDump(ret) + ret.asInstanceOf[Broadcast[T]] } catch { case ex: TimeoutException => logError(s"Could not execute broadcast in $timeout secs.", ex) @@ -501,6 +505,18 @@ abstract class GpuBroadcastExchangeExecBase( } } + // We have to do this explicitly here rather than similar to the general version one in + // [[GpuExec]] since in adaptive execution, the broadcast value has already been calculated + // before we tag this plan to dump. + private def doLoreDump(result: Broadcast[Any]): Unit = { + val inner = new SimpleRDD(session.sparkContext, result, schema) + getTagValue(LORE_DUMP_RDD_TAG).foreach { info => + val rdd = new GpuLoreDumpRDD(info, inner) + rdd.saveMeta() + rdd.foreach(_.close()) + } + } + override def runtimeStatistics: Statistics = { Statistics( sizeInBytes = metrics("dataSize").value, diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/execution/datasources/GpuWriteFiles.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/execution/datasources/GpuWriteFiles.scala index 7cc94359daa..f1ffcf4df1f 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/execution/datasources/GpuWriteFiles.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/execution/datasources/GpuWriteFiles.scala @@ -157,7 +157,7 @@ case class GpuWriteFilesExec( s" mismatch:\n$this") } - override protected def stringArgs: Iterator[Any] = Iterator(child) + override def stringArgs: Iterator[Any] = Iterator(child) } object GpuWriteFiles { diff --git a/tests/src/test/scala/com/nvidia/spark/rapids/lore/GpuLoreSuite.scala b/tests/src/test/scala/com/nvidia/spark/rapids/lore/GpuLoreSuite.scala new file mode 100644 index 00000000000..7db46718e89 --- /dev/null +++ b/tests/src/test/scala/com/nvidia/spark/rapids/lore/GpuLoreSuite.scala @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2020-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.lore + +import com.nvidia.spark.rapids.{FunSuiteWithTempDir, GpuColumnarToRowExec, RapidsConf, SparkQueryCompareTestSuite} +import org.apache.hadoop.fs.Path + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.{functions, DataFrame, SparkSession} +import org.apache.spark.sql.internal.SQLConf + +class GpuLoreSuite extends SparkQueryCompareTestSuite with FunSuiteWithTempDir with Logging { + test("Aggregate") { + doTestReplay("10[*]") { spark => + spark.range(0, 1000, 1, 100) + .selectExpr("id % 10 as key", "id % 100 as value") + .groupBy("key") + .agg(functions.sum("value").as("total")) + } + } + + test("Broadcast join") { + doTestReplay("32[*]") { spark => + val df1 = spark.range(0, 1000, 1, 10) + .selectExpr("id % 10 as key", "id % 100 as value") + .groupBy("key") + .agg(functions.sum("value").as("count")) + + val df2 = spark.range(0, 1000, 1, 10) + .selectExpr("(id % 10 + 5) as key", "id % 100 as value") + .groupBy("key") + .agg(functions.sum("value").as("count")) + + df1.join(df2, Seq("key")) + } + } + + test("Subquery Filter") { + doTestReplay("13[*]") { spark => + spark.range(0, 100, 1, 10) + .createTempView("df1") + + spark.range(50, 1000, 1, 10) + .createTempView("df2") + + spark.sql("select * from df1 where id > (select max(id) from df2)") + } + } + + test("Subquery in projection") { + doTestReplay("11[*]") { spark => + spark.sql( + """ + |CREATE TEMPORARY VIEW t1 + |AS SELECT * FROM VALUES + |(1, "a"), + |(2, "a"), + |(3, "a") t(id, value) + |""".stripMargin) + + spark.sql( + """ + |SELECT *, (SELECT COUNT(*) FROM t1) FROM t1 + |""".stripMargin) + } + } + + test("No broadcast join") { + doTestReplay("30[*]") { spark => + spark.conf.set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1") + + val df1 = spark.range(0, 1000, 1, 10) + .selectExpr("id % 10 as key", "id % 100 as value") + .groupBy("key") + .agg(functions.sum("value").as("count")) + + val df2 = spark.range(0, 1000, 1, 10) + .selectExpr("(id % 10 + 5) as key", "id % 100 as value") + .groupBy("key") + .agg(functions.sum("value").as("count")) + + df1.join(df2, Seq("key")) + } + } + + test("AQE broadcast") { + doTestReplay("90[*]") { spark => + spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true") + + val df1 = spark.range(0, 1000, 1, 10) + .selectExpr("id % 10 as key", "id % 100 as value") + .groupBy("key") + .agg(functions.sum("value").as("count")) + + val df2 = spark.range(0, 1000, 1, 10) + .selectExpr("(id % 10 + 5) as key", "id % 100 as value") + .groupBy("key") + .agg(functions.sum("value").as("count")) + + df1.join(df2, Seq("key")) + } + } + + test("AQE Exchange") { + doTestReplay("28[*]") { spark => + spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true") + + spark.range(0, 1000, 1, 100) + .selectExpr("id % 10 as key", "id % 100 as value") + .groupBy("key") + .agg(functions.sum("value").as("total")) + } + } + + test("Partition only") { + withGpuSparkSession{ spark => + spark.conf.set(RapidsConf.LORE_DUMP_PATH.key, TEST_FILES_ROOT.getAbsolutePath) + spark.conf.set(RapidsConf.LORE_DUMP_IDS.key, "3[0 2]") + + val df = spark.range(0, 1000, 1, 100) + .selectExpr("id % 10 as key", "id % 100 as value") + + val res = df.collect().length + println(s"Length of original: $res") + + + val restoredRes = GpuColumnarToRowExec(GpuLore.restoreGpuExec( + new Path(s"${TEST_FILES_ROOT.getAbsolutePath}/loreId-3"), spark)) + .executeCollect() + .length + + assert(20 == restoredRes) + } + } + + private def doTestReplay(loreDumpIds: String)(dfFunc: SparkSession => DataFrame) = { + val loreId = OutputLoreId.parse(loreDumpIds).head._1 + withGpuSparkSession { spark => + spark.conf.set(RapidsConf.LORE_DUMP_PATH.key, TEST_FILES_ROOT.getAbsolutePath) + spark.conf.set(RapidsConf.LORE_DUMP_IDS.key, loreDumpIds) + + val df = dfFunc(spark) + + val expectedLength = df.collect().length + + val restoredResultLength = GpuColumnarToRowExec(GpuLore.restoreGpuExec( + new Path(s"${TEST_FILES_ROOT.getAbsolutePath}/loreId-$loreId"), + spark)) + .executeCollect() + .length + + assert(expectedLength == restoredResultLength) + } + } +} diff --git a/tests/src/test/scala/com/nvidia/spark/rapids/lore/OutputLoreIdSuite.scala b/tests/src/test/scala/com/nvidia/spark/rapids/lore/OutputLoreIdSuite.scala new file mode 100644 index 00000000000..aad3d997b9d --- /dev/null +++ b/tests/src/test/scala/com/nvidia/spark/rapids/lore/OutputLoreIdSuite.scala @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2020-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.lore + +import org.scalatest.funsuite.AnyFunSuite + +class OutputLoreIdSuite extends AnyFunSuite { + test("Parse one output lore id") { + val expectedLoreIds = Map(1 -> OutputLoreId(1, Set(1, 2, 4, 8))) + val loreIds = OutputLoreId.parse("1[1 2 4 8]") + + assert(loreIds == expectedLoreIds) + } + + test("Parse multi output lore id") { + val expectedLoreIds = Map( + 1 -> OutputLoreId(1, Set(1, 2, 4, 8)), + 2 -> OutputLoreId(2, Set(1, 4, 5, 6, 7, 8, 100)) + ) + val loreIds = OutputLoreId.parse("1[1 2 4 8], 2[1 4-9 100]") + + assert(loreIds == expectedLoreIds) + } + + test("Parse empty output lore id should fail") { + assertThrows[IllegalArgumentException] { + OutputLoreId.parse(" 1, 2 ") + } + } + + test("Parse mixed") { + val expectedLoreIds = Map( + 1 -> OutputLoreId(1), + 2 -> OutputLoreId(2, Set(4, 5, 8)), + 3 -> OutputLoreId(3, Set(1, 2, 4, 8)) + ) + val loreIds = OutputLoreId.parse("1[*], 2[4-6 8] , 3[1 2 4 8]") + + assert(loreIds == expectedLoreIds) + } +} From ba64999c770e82276c1e57cb056a70ee6053b6d8 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Tue, 2 Jul 2024 14:51:08 +0800 Subject: [PATCH 57/79] Update Scala2.13 premerge CI against JDK17 (#11117) To fix issue: https://github.com/NVIDIA/spark-rapids/issues/11113 To support Spark 4.0+ shims, we change scala2.13 build and test against JDK17. Signed-off-by: Tim Liu --- jenkins/Dockerfile-blossom.ubuntu | 4 ++-- jenkins/spark-premerge-build.sh | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/jenkins/Dockerfile-blossom.ubuntu b/jenkins/Dockerfile-blossom.ubuntu index c2c2dabf293..755dd3eeaa0 100644 --- a/jenkins/Dockerfile-blossom.ubuntu +++ b/jenkins/Dockerfile-blossom.ubuntu @@ -40,7 +40,7 @@ ARG UCX_VER ARG ARCH=amd64 ARG UCX_ARCH=x86_64 -# Install jdk-8, jdk-11, maven, docker image +# Install jdk-8, jdk-11, jdk-17, maven, docker image RUN apt-get update -y && \ apt-get install -y software-properties-common rsync @@ -48,7 +48,7 @@ RUN apt-get update -y && \ RUN add-apt-repository ppa:deadsnakes/ppa && \ apt-get update -y && \ DEBIAN_FRONTEND="noninteractive" apt-get install -y maven \ - openjdk-8-jdk openjdk-11-jdk python3.9 python3.9-distutils python3-setuptools tzdata git zip unzip wget \ + openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk python3.9 python3.9-distutils python3-setuptools tzdata git zip unzip wget \ inetutils-ping expect wget libnuma1 libgomp1 locales # apt python3-pip would install pip for OS default python3 version only diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh index ec4b74cc902..e81db74cbd4 100755 --- a/jenkins/spark-premerge-build.sh +++ b/jenkins/spark-premerge-build.sh @@ -183,6 +183,11 @@ ci_2() { ci_scala213() { echo "Run premerge ci (Scala 2.13) testing..." + # Run scala2.13 build and test against JDK17 + export JAVA_HOME=$(echo /usr/lib/jvm/java-1.17.0-*) + update-java-alternatives --set $JAVA_HOME + java -version + cd scala2.13 ln -sf ../jenkins jenkins From e92cbd23d100558c2c13794163aa4374bc37d088 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Tue, 2 Jul 2024 13:26:52 -0500 Subject: [PATCH 58/79] Profiler: Disable collecting async allocation events by default (#10965) Signed-off-by: Jason Lowe --- .../scala/com/nvidia/spark/rapids/RapidsConf.scala | 10 +++++++++- .../main/scala/com/nvidia/spark/rapids/profiler.scala | 7 ++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index 406d09a7a32..ba6ec25aeba 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -745,6 +745,12 @@ val GPU_COREDUMP_PIPE_PATTERN = conf("spark.rapids.gpu.coreDump.pipePattern") .stringConf .createOptional + val PROFILE_ASYNC_ALLOC_CAPTURE = conf("spark.rapids.profile.asyncAllocCapture") + .doc("Whether the profiler should capture async CUDA allocation and free events") + .internal() + .booleanConf + .createWithDefault(false) + val PROFILE_DRIVER_POLL_MILLIS = conf("spark.rapids.profile.driverPollMillis") .doc("Interval in milliseconds the executors will poll for job and stage completion when " + "stage-level profiling is used.") @@ -772,7 +778,7 @@ val GPU_COREDUMP_PIPE_PATTERN = conf("spark.rapids.gpu.coreDump.pipePattern") .doc("Buffer size to use when writing profile records.") .internal() .bytesConf(ByteUnit.BYTE) - .createWithDefault(1024 * 1024) + .createWithDefault(8 * 1024 * 1024) // ENABLE/DISABLE PROCESSING @@ -2531,6 +2537,8 @@ class RapidsConf(conf: Map[String, String]) extends Logging { lazy val profileDriverPollMillis: Int = get(PROFILE_DRIVER_POLL_MILLIS) + lazy val profileAsyncAllocCapture: Boolean = get(PROFILE_ASYNC_ALLOC_CAPTURE) + lazy val profileCompression: String = get(PROFILE_COMPRESSION) lazy val profileFlushPeriodMillis: Int = get(PROFILE_FLUSH_PERIOD_MILLIS) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/profiler.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/profiler.scala index e6e2bcc9f7d..924a75a7b65 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/profiler.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/profiler.scala @@ -80,7 +80,12 @@ object ProfilerOnExecutor extends Logging { case c => Some(TrampolineUtil.createCodec(pluginCtx.conf(), c)) } val w = new ProfileWriter(pluginCtx, pathPrefix, codec) - Profiler.init(w, conf.profileWriteBufferSize, conf.profileFlushPeriodMillis) + val profilerConf = new Profiler.Config.Builder() + .withWriteBufferSize(conf.profileWriteBufferSize) + .withFlushPeriodMillis(conf.profileFlushPeriodMillis) + .withAllocAsyncCapturing(conf.profileAsyncAllocCapture) + .build() + Profiler.init(w, profilerConf) Some(w) } else { None From 6cd094e3feb150c4a538497fdfb03d84ad8b0602 Mon Sep 17 00:00:00 2001 From: Navin Kumar <97137715+NVnavkumar@users.noreply.github.com> Date: Wed, 3 Jul 2024 07:18:12 -0700 Subject: [PATCH 59/79] Dataproc serverless test fixes (#11043) * Skip cast tests that throw exceptions on CPU Signed-off-by: Navin Kumar * This Exec doesn't actually run on the GPU so this should be added to this mark Signed-off-by: Navin Kumar * Update README.md with dataproc_serverless runtime_env value Signed-off-by: Navin Kumar --------- Signed-off-by: Navin Kumar --- integration_tests/README.md | 2 +- integration_tests/src/main/python/cast_test.py | 8 ++++++-- integration_tests/src/main/python/conftest.py | 3 +++ integration_tests/src/main/python/join_test.py | 4 ++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/integration_tests/README.md b/integration_tests/README.md index c5f55f9f605..504efb24563 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -296,7 +296,7 @@ You do need to have access to a compatible GPU with the needed CUDA drivers. The ### Runtime Environment -`--runtime_env` is used to specify the environment you are running the tests in. Valid values are `databricks`,`emr`,`dataproc` and `apache`. This is generally used +`--runtime_env` is used to specify the environment you are running the tests in. Valid values are `databricks`,`emr`,`dataproc`,`dataproc_serverless` and `apache`. This is generally used when certain environments have different behavior, and the tests don't have a good way to auto-detect the environment yet. ### timezone diff --git a/integration_tests/src/main/python/cast_test.py b/integration_tests/src/main/python/cast_test.py index a56afe29ab9..11bb42fe9f7 100644 --- a/integration_tests/src/main/python/cast_test.py +++ b/integration_tests/src/main/python/cast_test.py @@ -15,7 +15,7 @@ import pytest from asserts import * -from conftest import is_not_utc, is_supported_time_zone +from conftest import is_not_utc, is_supported_time_zone, is_dataproc_serverless_runtime from data_gen import * from spark_session import * from marks import allow_non_gpu, approximate_float, datagen_overrides, tz_sensitive_test @@ -181,7 +181,11 @@ def test_cast_string_timestamp_fallback(): @approximate_float @pytest.mark.parametrize('data_gen', [ - decimal_gen_32bit, decimal_gen_32bit_neg_scale, DecimalGen(precision=7, scale=7), + decimal_gen_32bit, + pytest.param(decimal_gen_32bit_neg_scale, marks= + pytest.mark.skipif(is_dataproc_serverless_runtime(), + reason="Dataproc Serverless does not support negative scale for Decimal cast")), + DecimalGen(precision=7, scale=7), decimal_gen_64bit, decimal_gen_128bit, DecimalGen(precision=30, scale=2), DecimalGen(precision=36, scale=5), DecimalGen(precision=38, scale=0), DecimalGen(precision=38, scale=10), DecimalGen(precision=36, scale=-5), diff --git a/integration_tests/src/main/python/conftest.py b/integration_tests/src/main/python/conftest.py index 6af40b99768..49f69f31837 100644 --- a/integration_tests/src/main/python/conftest.py +++ b/integration_tests/src/main/python/conftest.py @@ -87,6 +87,9 @@ def is_emr_runtime(): def is_dataproc_runtime(): return runtime_env() == "dataproc" +def is_dataproc_serverless_runtime(): + return runtime_env() == "dataproc_serverless" + def get_test_tz(): return os.environ.get('TZ', 'UTC') diff --git a/integration_tests/src/main/python/join_test.py b/integration_tests/src/main/python/join_test.py index ef9f4d03f24..35e837cb436 100644 --- a/integration_tests/src/main/python/join_test.py +++ b/integration_tests/src/main/python/join_test.py @@ -1109,7 +1109,7 @@ def do_join(spark): @ignore_order(local=True) -@allow_non_gpu("ProjectExec", "FilterExec", "BroadcastHashJoinExec", "ColumnarToRowExec", "BroadcastExchangeExec") +@allow_non_gpu("ProjectExec", "FilterExec", "BroadcastHashJoinExec", "ColumnarToRowExec", "BroadcastExchangeExec", "BatchScanExec") @pytest.mark.parametrize("disable_build", [True, False]) def test_broadcast_hash_join_fix_fallback_by_inputfile(spark_tmp_path, disable_build): data_path_parquet = spark_tmp_path + "/parquet" @@ -1145,7 +1145,7 @@ def do_join(spark): @ignore_order(local=True) -@allow_non_gpu("ProjectExec", "BroadcastNestedLoopJoinExec", "ColumnarToRowExec", "BroadcastExchangeExec") +@allow_non_gpu("ProjectExec", "BroadcastNestedLoopJoinExec", "ColumnarToRowExec", "BroadcastExchangeExec", "BatchScanExec") @pytest.mark.parametrize("disable_build", [True, False]) def test_broadcast_nested_join_fix_fallback_by_inputfile(spark_tmp_path, disable_build): data_path_parquet = spark_tmp_path + "/parquet" From 5635fd4dbc3da8d15fb5e6a1610db5e59dcfafdc Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Wed, 3 Jul 2024 16:03:19 -0500 Subject: [PATCH 60/79] Fix issue with DPP and AQE on reused broadcast exchanges [databricks] (#11118) Signed-off-by: Robert (Bobby) Evans --- integration_tests/src/main/python/aqe_test.py | 58 ++++++++++++++++++- .../spark/rapids/GpuTransitionOverrides.scala | 21 ++++++- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/integration_tests/src/main/python/aqe_test.py b/integration_tests/src/main/python/aqe_test.py index ba0553912d4..e16a4eafcab 100755 --- a/integration_tests/src/main/python/aqe_test.py +++ b/integration_tests/src/main/python/aqe_test.py @@ -19,7 +19,7 @@ from conftest import is_databricks_runtime, is_not_utc from data_gen import * from marks import ignore_order, allow_non_gpu -from spark_session import with_cpu_session, is_databricks113_or_later +from spark_session import with_cpu_session, is_databricks113_or_later, is_before_spark_330 # allow non gpu when time zone is non-UTC because of https://github.com/NVIDIA/spark-rapids/issues/9653' not_utc_aqe_allow=['ShuffleExchangeExec', 'HashAggregateExec'] if is_not_utc() else [] @@ -335,3 +335,59 @@ def do_it(spark): non_exist_classes="GpuBroadcastExchangeExec", conf=conf) + +# this should be fixed by https://github.com/NVIDIA/spark-rapids/issues/11120 +aqe_join_with_dpp_fallback=["FilterExec"] if (is_databricks_runtime() or is_before_spark_330()) else [] + +# Verify that DPP and AQE can coexist in even some odd cases involving multiple tables +@ignore_order(local=True) +@allow_non_gpu(*aqe_join_with_dpp_fallback) +def test_aqe_join_with_dpp(spark_tmp_path): + data_path = spark_tmp_path + '/PARQUET_DATA' + def write_data(spark): + spark.range(34).selectExpr("concat('test_', id % 9) as test_id", + "concat('site_', id % 2) as site_id").repartition(200).write.parquet(data_path + "/tests") + spark.range(1000).selectExpr( + "CAST(if (id % 2 == 0, '1990-01-01', '1990-01-02') as DATE) as day", + "concat('test_', (id % 9) + 100) as test_id", + "concat('site_', id % 40) as site_id", + "rand(1) as extra_info").write.partitionBy("day", "site_id", "test_id").parquet(data_path + "/infoA") + spark.range(1000).selectExpr( + "CAST(if (id % 2 == 0, '1990-01-01', '1990-01-02') as DATE) as day", + "concat('exp_', id % 9) as test_spec", + "concat('LONG_SITE_NAME_', id % 40) as site_id", + "rand(0) as extra_info").write.partitionBy("day", "site_id").parquet(data_path + "/infoB") + + with_cpu_session(write_data) + + def run_test(spark): + spark.read.parquet(data_path + "/tests/").createOrReplaceTempView("tests") + spark.read.parquet(data_path + "/infoA/").createOrReplaceTempView("infoA") + spark.read.parquet(data_path + "/infoB/").createOrReplaceTempView("infoB") + return spark.sql(""" + with tmp as + (SELECT + site_id, + day, + test_id + FROM + infoA + UNION ALL + SELECT + CASE + WHEN site_id = 'LONG_SITE_NAME_0' then 'site_0' + WHEN site_id = 'LONG_SITE_NAME_1' then 'site_1' + ELSE site_id + END AS site_id, + day, + test_spec AS test_id + FROM infoB + ) + SELECT * + FROM tmp a + JOIN tests b ON a.test_id = b.test_id AND a.site_id = b.site_id + WHERE day = '1990-01-01' + AND a.site_id IN ('site_0', 'site_1') + """) + + assert_gpu_and_cpu_are_equal_collect(run_test, conf=_adaptive_conf) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuTransitionOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuTransitionOverrides.scala index c8596f983d9..00192ed775c 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuTransitionOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuTransitionOverrides.scala @@ -34,7 +34,7 @@ import org.apache.spark.sql.execution.adaptive._ import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec import org.apache.spark.sql.execution.command.{DataWritingCommandExec, ExecutedCommandExec} import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2ScanExecBase, DropTableExec, ShowTablesExec} -import org.apache.spark.sql.execution.exchange.{BroadcastExchangeLike, Exchange, ReusedExchangeExec, ShuffleExchangeLike} +import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, BroadcastExchangeLike, Exchange, ReusedExchangeExec, ShuffleExchangeLike} import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec, HashedRelationBroadcastMode} import org.apache.spark.sql.rapids.{GpuDataSourceScanExec, GpuFileSourceScanExec, GpuShuffleEnv, GpuTaskMetrics} import org.apache.spark.sql.rapids.execution.{ExchangeMappingCache, GpuBroadcastExchangeExec, GpuBroadcastExchangeExecBase, GpuBroadcastToRowExec, GpuCustomShuffleReaderExec, GpuHashJoin, GpuShuffleExchangeExecBase} @@ -202,6 +202,12 @@ class GpuTransitionOverrides extends Rule[SparkPlan] { SparkShimImpl.addRowShuffleToQueryStageTransitionIfNeeded(c2r, e) case ColumnarToRowExec(e: BroadcastQueryStageExec) => + // In Spark an AdaptiveSparkPlanExec when doing a broadcast expects to only ever see + // a BroadcastQueryStageExec as its child. The ColumnarToRowExec can be inserted by Spark + // when it sees us doing columnar processing and the output expects rows. But we cannot + // keep the ColumnarToRowExec there, nor can we insert in a GpuBroadcastToRowExec in its + // place. We have to rearrange the order of operations so it is a child of the + // BroadcastQueryStageExec (which will likely involve another broadcast) e.plan match { case ReusedExchangeExec(output, b: GpuBroadcastExchangeExec) => // we can't directly re-use a GPU broadcast exchange to feed a CPU broadcast @@ -212,8 +218,17 @@ class GpuTransitionOverrides extends Rule[SparkPlan] { case IdentityBroadcastMode => (Some(0), None) case m => throw new UnsupportedOperationException(s"Unknown broadcast mode $m") } - GpuBroadcastToRowExec(keys, b.mode, e)(index, keyExprs) - case _ => GpuColumnarToRowExec(optimizeAdaptiveTransitions(e, Some(plan))) + SparkShimImpl.newBroadcastQueryStageExec(e, + GpuBroadcastToRowExec(keys, b.mode, e.plan)(index, keyExprs)) + case b: GpuBroadcastExchangeExec => + // This should never happen as AQE with a BroadcastQueryStageExec should + // only show up on a reused exchange, but just in case we try to do the right + // thing here + SparkShimImpl.newBroadcastQueryStageExec(e, + BroadcastExchangeExec(b.mode, GpuColumnarToRowExec(b))) + case other => + throw new IllegalStateException(s"Don't know how to handle a " + + s"BroadcastQueryStageExec with $other") } // inserts postColumnarToRowTransition into newly-created GpuColumnarToRowExec From 6fa51ba404a239d400e65c38bb7c95235b3e7a18 Mon Sep 17 00:00:00 2001 From: MithunR Date: Wed, 3 Jul 2024 16:06:16 -0700 Subject: [PATCH 61/79] Fix miscellaneous integ tests for Spark 4 [databricks] (#11097) * Prep miscellaneous integration tests for Spark 4 Fixes #11020. (grouping_sets_test.py) Fixes #11023. (dpp_test.py) Fixes #11025. (date_time_test.py) Fixes #11026. (map_test.py) This commit prepares miscellaneous integration tests to be run on Spark 4. Certain integration tests fail on Spark 4 because of ANSI mode being enabled by default. This commit disables ANSI on the failing tests, or introduces other fixes so that the tests may pass correctly. Signed-off-by: MithunR --- integration_tests/src/main/python/date_time_test.py | 9 ++++++++- integration_tests/src/main/python/dpp_test.py | 12 ++++++++++-- .../src/main/python/grouping_sets_test.py | 4 +++- integration_tests/src/main/python/map_test.py | 9 +++++++-- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/integration_tests/src/main/python/date_time_test.py b/integration_tests/src/main/python/date_time_test.py index 9e2e98006ab..0c877f00238 100644 --- a/integration_tests/src/main/python/date_time_test.py +++ b/integration_tests/src/main/python/date_time_test.py @@ -17,7 +17,7 @@ from conftest import is_utc, is_supported_time_zone, get_test_tz from data_gen import * from datetime import date, datetime, timezone -from marks import ignore_order, incompat, allow_non_gpu, datagen_overrides, tz_sensitive_test +from marks import allow_non_gpu, datagen_overrides, disable_ansi_mode, ignore_order, incompat, tz_sensitive_test from pyspark.sql.types import * from spark_session import with_cpu_session, is_before_spark_330, is_before_spark_350 import pyspark.sql.functions as f @@ -91,6 +91,8 @@ def fun(spark): assert_gpu_and_cpu_are_equal_collect(fun) + +@disable_ansi_mode # ANSI mode tested separately. # Should specify `spark.sql.legacy.interval.enabled` to test `DateAddInterval` after Spark 3.2.0, # refer to https://issues.apache.org/jira/browse/SPARK-34896 # [SPARK-34896][SQL] Return day-time interval from dates subtraction @@ -437,6 +439,8 @@ def test_string_unix_timestamp_ansi_exception(): error_message="Exception", conf=ansi_enabled_conf) + +@disable_ansi_mode # ANSI mode is tested separately. @tz_sensitive_test @pytest.mark.skipif(not is_supported_time_zone(), reason="not all time zones are supported now, refer to https://github.com/NVIDIA/spark-rapids/issues/6839, please update after all time zones are supported") @pytest.mark.parametrize('parser_policy', ["CORRECTED", "EXCEPTION"], ids=idfn) @@ -561,6 +565,8 @@ def test_date_format_maybe_incompat(data_gen, date_format): assert_gpu_and_cpu_are_equal_collect( lambda spark : unary_op_df(spark, data_gen).selectExpr("date_format(a, '{}')".format(date_format)), conf) + +@disable_ansi_mode # ANSI mode tested separately. # Reproduce conditions for https://github.com/NVIDIA/spark-rapids/issues/5670 # where we had a failure due to GpuCast canonicalization with timezone. # In this case it was doing filter after project, the way I get that to happen is by adding in the @@ -594,6 +600,7 @@ def test_unsupported_fallback_date_format(data_gen): conf) +@disable_ansi_mode # Failure cases for ANSI mode are tested separately. @allow_non_gpu('ProjectExec') def test_unsupported_fallback_to_date(): date_gen = StringGen(pattern="2023-08-01") diff --git a/integration_tests/src/main/python/dpp_test.py b/integration_tests/src/main/python/dpp_test.py index cd4610cf95c..b362a4175f3 100644 --- a/integration_tests/src/main/python/dpp_test.py +++ b/integration_tests/src/main/python/dpp_test.py @@ -19,7 +19,7 @@ from asserts import assert_cpu_and_gpu_are_equal_collect_with_capture, assert_gpu_and_cpu_are_equal_collect from conftest import spark_tmp_table_factory from data_gen import * -from marks import ignore_order, allow_non_gpu, datagen_overrides +from marks import ignore_order, allow_non_gpu, datagen_overrides, disable_ansi_mode from spark_session import is_before_spark_320, with_cpu_session, is_before_spark_312, is_databricks_runtime, is_databricks113_or_later # non-positive values here can produce a degenerative join, so here we ensure that most values are @@ -167,7 +167,7 @@ def fn(spark): ''' ] - +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # When BroadcastExchangeExec is available on filtering side, and it can be reused: # DynamicPruningExpression(InSubqueryExec(value, GpuSubqueryBroadcastExec))) @ignore_order @@ -198,6 +198,7 @@ def test_dpp_reuse_broadcast_exchange(spark_tmp_table_factory, store_format, s_i conf=dict(_exchange_reuse_conf + [('spark.sql.adaptive.enabled', aqe_enabled)])) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # The SubqueryBroadcast can work on GPU even if the scan who holds it fallbacks into CPU. @ignore_order @pytest.mark.allow_non_gpu('FileSourceScanExec') @@ -215,6 +216,7 @@ def test_dpp_reuse_broadcast_exchange_cpu_scan(spark_tmp_table_factory): ('spark.rapids.sql.format.parquet.read.enabled', 'false')])) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # When BroadcastExchange is not available and non-broadcast DPPs are forbidden, Spark will bypass it: # DynamicPruningExpression(Literal.TrueLiteral) @ignore_order @@ -238,6 +240,7 @@ def test_dpp_bypass(spark_tmp_table_factory, store_format, s_index, aqe_enabled) conf=dict(_bypass_conf + [('spark.sql.adaptive.enabled', aqe_enabled)])) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # When BroadcastExchange is not available, but it is still worthwhile to run DPP, # then Spark will plan an extra Aggregate to collect filtering values: # DynamicPruningExpression(InSubqueryExec(value, SubqueryExec(Aggregate(...)))) @@ -261,6 +264,7 @@ def test_dpp_via_aggregate_subquery(spark_tmp_table_factory, store_format, s_ind conf=dict(_no_exchange_reuse_conf + [('spark.sql.adaptive.enabled', aqe_enabled)])) +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # When BroadcastExchange is not available, Spark will skip DPP if there is no potential benefit @ignore_order @pytest.mark.parametrize('store_format', ['parquet', 'orc'], ids=idfn) @@ -321,6 +325,8 @@ def create_dim_table_for_like(spark): exist_classes, conf=dict(_exchange_reuse_conf + [('spark.sql.adaptive.enabled', aqe_enabled)])) + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # Test handling DPP expressions from a HashedRelation that rearranges columns @pytest.mark.parametrize('aqe_enabled', [ 'false', @@ -351,6 +357,8 @@ def setup_tables(spark): ("spark.rapids.sql.castStringToTimestamp.enabled", "true"), ("spark.rapids.sql.hasExtendedYearValues", "false")])) + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # Test handling DPP subquery that could broadcast EmptyRelation rather than a GPU serialized batch @pytest.mark.parametrize('aqe_enabled', [ 'false', diff --git a/integration_tests/src/main/python/grouping_sets_test.py b/integration_tests/src/main/python/grouping_sets_test.py index 24f8dd1810c..1821efa5c7a 100644 --- a/integration_tests/src/main/python/grouping_sets_test.py +++ b/integration_tests/src/main/python/grouping_sets_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -41,6 +41,8 @@ 'GROUP BY a, GROUPING SETS((a, b), (a), (), (a, b), (a), (b), ())', ] + +@disable_ansi_mode # https://github.com/NVIDIA/spark-rapids/issues/5114 # test nested syntax of grouping set, rollup and cube @ignore_order @pytest.mark.parametrize('data_gen', [_grouping_set_gen], ids=idfn) diff --git a/integration_tests/src/main/python/map_test.py b/integration_tests/src/main/python/map_test.py index d5e49d5eb65..72e92280d43 100644 --- a/integration_tests/src/main/python/map_test.py +++ b/integration_tests/src/main/python/map_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ from conftest import is_not_utc from data_gen import * from conftest import is_databricks_runtime -from marks import allow_non_gpu, ignore_order, datagen_overrides +from marks import allow_non_gpu, datagen_overrides, disable_ansi_mode, ignore_order from spark_session import * from pyspark.sql.functions import create_map, col, lit, row_number from pyspark.sql.types import * @@ -138,6 +138,7 @@ def test_get_map_value_string_keys(data_gen): for key in numeric_key_gens for value in get_map_value_gens()] +@disable_ansi_mode # ANSI mode failures are tested separately. @pytest.mark.parametrize('data_gen', numeric_key_map_gens, ids=idfn) def test_get_map_value_numeric_keys(data_gen): key_gen = data_gen._key_gen @@ -151,6 +152,7 @@ def test_get_map_value_numeric_keys(data_gen): 'a[999]')) +@disable_ansi_mode # ANSI mode failures are tested separately. @pytest.mark.parametrize('data_gen', supported_key_map_gens, ids=idfn) @allow_non_gpu(*non_utc_allow) def test_get_map_value_supported_keys(data_gen): @@ -174,6 +176,7 @@ def test_get_map_value_fallback_keys(data_gen): cpu_fallback_class_name="GetMapValue") +@disable_ansi_mode # ANSI mode failures are tested separately. @pytest.mark.parametrize('key_gen', numeric_key_gens, ids=idfn) def test_basic_scalar_map_get_map_value(key_gen): def query_map_scalar(spark): @@ -639,6 +642,8 @@ def test_map_element_at_ansi_null(data_gen): 'element_at(a, "NOT_FOUND")'), conf=ansi_enabled_conf) + +@disable_ansi_mode # ANSI mode failures are tested separately. @pytest.mark.parametrize('data_gen', map_gens_sample, ids=idfn) @allow_non_gpu(*non_utc_allow) def test_transform_values(data_gen): From b98b03fd0e0af481a4c061b2cfcd70143107047b Mon Sep 17 00:00:00 2001 From: MithunR Date: Wed, 3 Jul 2024 16:07:35 -0700 Subject: [PATCH 62/79] Fix test_window_group_limits_fallback. (#11133) Fixes #11119. `test_window_group_limits_fallback_for_row_number` can fail non-deterministically when run against a multi-node Spark cluster. This is because the ordering of the input is non-deterministic when multiple null rows are included. The tests have been changed for deterministic ordering, by including a unique order by column. Signed-off-by: MithunR --- integration_tests/src/main/python/window_function_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/src/main/python/window_function_test.py b/integration_tests/src/main/python/window_function_test.py index 44bc2a07d57..653eaffa940 100644 --- a/integration_tests/src/main/python/window_function_test.py +++ b/integration_tests/src/main/python/window_function_test.py @@ -2102,8 +2102,8 @@ def assert_query_runs_on(exec, conf): ], ids=idfn) @pytest.mark.parametrize('rank_clause', [ - 'RANK() OVER (PARTITION BY a ORDER BY b) ', - 'DENSE_RANK() OVER (PARTITION BY a ORDER BY b) ', + 'RANK() OVER (PARTITION BY a ORDER BY b, c) ', + 'DENSE_RANK() OVER (PARTITION BY a ORDER BY b, c) ', 'RANK() OVER (ORDER BY a,b,c) ', 'DENSE_RANK() OVER (ORDER BY a,b,c) ', ]) @@ -2151,7 +2151,7 @@ def test_window_group_limits_fallback_for_row_number(): data_gen = _grpkey_longs_with_no_nulls query = """ SELECT * FROM ( - SELECT *, ROW_NUMBER() OVER (PARTITION BY a ORDER BY b) AS rnk + SELECT *, ROW_NUMBER() OVER (PARTITION BY a ORDER BY b, c) AS rnk FROM window_agg_table ) WHERE rnk < 3 From c592d731d38b6abfe3719d9e7810e57d8896dcd0 Mon Sep 17 00:00:00 2001 From: "Hongbin Ma (Mahone)" Date: Thu, 4 Jul 2024 14:48:12 +0800 Subject: [PATCH 63/79] Improve MetricsSuite to allow more gc jitter [databricks] (#11139) * improve MetricsSuite to allow more gc jitter Signed-off-by: Hongbin Ma (Mahone) * fix comment Signed-off-by: Hongbin Ma (Mahone) --------- Signed-off-by: Hongbin Ma (Mahone) --- .../nvidia/spark/rapids/MetricsSuite.scala | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/src/test/scala/com/nvidia/spark/rapids/MetricsSuite.scala b/tests/src/test/scala/com/nvidia/spark/rapids/MetricsSuite.scala index 580c5a2ed55..f5522454052 100644 --- a/tests/src/test/scala/com/nvidia/spark/rapids/MetricsSuite.scala +++ b/tests/src/test/scala/com/nvidia/spark/rapids/MetricsSuite.scala @@ -26,12 +26,12 @@ class MetricsSuite extends AnyFunSuite { val m1 = new LocalGpuMetric() m1.ns( m1.ns( - Thread.sleep(100) + Thread.sleep(1000) ) ) - // if the timing is duplicated, the value should be around 200,000,000 - assert(m1.value < 100000000 * 1.5) - assert(m1.value > 100000000 * 0.5) + // if the timing is duplicated, the value should be around 2,000,000,000 + assert(m1.value < 1000000000 * 1.9) + assert(m1.value > 1000000000 * 0.5) } test("MetricRange: duplicate timing on the same metrics") { @@ -39,15 +39,15 @@ class MetricsSuite extends AnyFunSuite { val m2 = new LocalGpuMetric() withResource(new MetricRange(m1, m2)) { _ => withResource(new MetricRange(m2, m1)) { _ => - Thread.sleep(100) + Thread.sleep(1000) } } - // if the timing is duplicated, the value should be around 200,000,000 - assert(m1.value < 100000000 * 1.5) - assert(m1.value > 100000000 * 0.5) - assert(m2.value < 100000000 * 1.5) - assert(m2.value > 100000000 * 0.5) + // if the timing is duplicated, the value should be around 2,000,000,000 + assert(m1.value < 1000000000 * 1.9) + assert(m1.value > 1000000000 * 0.5) + assert(m2.value < 1000000000 * 1.9) + assert(m2.value > 1000000000 * 0.5) } test("NvtxWithMetrics: duplicate timing on the same metrics") { @@ -55,14 +55,14 @@ class MetricsSuite extends AnyFunSuite { val m2 = new LocalGpuMetric() withResource(new NvtxWithMetrics("a", NvtxColor.BLUE, m1, m2)) { _ => withResource(new NvtxWithMetrics("b", NvtxColor.BLUE, m2, m1)) { _ => - Thread.sleep(100) + Thread.sleep(1000) } } - // if the timing is duplicated, the value should be around 200,000,000 - assert(m1.value < 100000000 * 1.5) - assert(m1.value > 100000000 * 0.5) - assert(m2.value < 100000000 * 1.5) - assert(m2.value > 100000000 * 0.5) + // if the timing is duplicated, the value should be around 2,000,000,000 + assert(m1.value < 1000000000 * 1.9) + assert(m1.value > 1000000000 * 0.5) + assert(m2.value < 1000000000 * 1.9) + assert(m2.value > 1000000000 * 0.5) } } From 2ffaf94e28292cfe79410e28d0565f2222fd65b2 Mon Sep 17 00:00:00 2001 From: Liangcai Li Date: Fri, 5 Jul 2024 08:43:22 +0800 Subject: [PATCH 64/79] Add `HiveHash` support on GPU (#11094) This PR adds the HiveHash support on GPU with some common types, and supported types are [bool, byte, short, int, long, string, date, timestamp, float, double]. It also supports bucketed write for the write commands leveraging HiveHash to generate bucket IDs. --------- Signed-off-by: Firestarman --- .../advanced_configs.md | 1 + docs/supported_ops.md | 1623 +++++++++-------- .../main/python/datasourcev2_write_test.py | 95 +- .../main/python/hive_parquet_write_test.py | 112 +- .../src/main/python/hive_write_test.py | 44 +- .../nvidia/spark/rapids/GpuOverrides.scala | 14 +- .../com/nvidia/spark/rapids/RapidsConf.scala | 11 + .../sql/hive/rapids/GpuHiveFileFormat.scala | 6 +- .../sql/hive/rapids/GpuSaveAsHiveFile.scala | 4 +- .../sql/rapids/GpuFileFormatDataWriter.scala | 33 +- ...GpuInsertIntoHadoopFsRelationCommand.scala | 5 +- .../spark/sql/rapids/HashFunctions.scala | 17 +- .../rapids/shims/BucketSpecForHiveShim.scala | 43 + ...ngUtils.scala => BucketingUtilsShim.scala} | 10 +- ...dCreateHiveTableAsSelectCommandShims.scala | 4 +- .../GpuCreateHiveTableAsSelectCommand.scala | 6 +- .../rapids/shims/GpuInsertIntoHiveTable.scala | 9 +- .../sql/rapids/GpuFileFormatWriter.scala | 7 +- .../rapids/shims/BucketSpecForHiveShim.scala | 36 + ...ngUtils.scala => BucketingUtilsShim.scala} | 14 +- .../rapids/shims/BucketSpecForHiveShim.scala | 30 + .../rapids/shims/GpuInsertIntoHiveTable.scala | 8 +- ...dCreateHiveTableAsSelectCommandShims.scala | 3 +- .../sql/rapids/GpuFileFormatWriter.scala | 7 +- .../spark/rapids/HiveHashTestSuite.scala | 181 ++ tools/generated_files/311/operatorsScore.csv | 1 + tools/generated_files/311/supportedExprs.csv | 2 + tools/generated_files/312/operatorsScore.csv | 1 + tools/generated_files/312/supportedExprs.csv | 2 + tools/generated_files/313/operatorsScore.csv | 1 + tools/generated_files/313/supportedExprs.csv | 2 + tools/generated_files/320/operatorsScore.csv | 1 + tools/generated_files/320/supportedExprs.csv | 2 + tools/generated_files/321/operatorsScore.csv | 1 + tools/generated_files/321/supportedExprs.csv | 2 + .../generated_files/321cdh/operatorsScore.csv | 1 + .../generated_files/321cdh/supportedExprs.csv | 2 + tools/generated_files/322/operatorsScore.csv | 1 + tools/generated_files/322/supportedExprs.csv | 2 + tools/generated_files/323/operatorsScore.csv | 1 + tools/generated_files/323/supportedExprs.csv | 2 + tools/generated_files/324/operatorsScore.csv | 1 + tools/generated_files/324/supportedExprs.csv | 2 + tools/generated_files/330/operatorsScore.csv | 1 + tools/generated_files/330/supportedExprs.csv | 2 + .../generated_files/330cdh/operatorsScore.csv | 1 + .../generated_files/330cdh/supportedExprs.csv | 2 + tools/generated_files/331/operatorsScore.csv | 1 + tools/generated_files/331/supportedExprs.csv | 2 + tools/generated_files/332/operatorsScore.csv | 1 + tools/generated_files/332/supportedExprs.csv | 2 + .../generated_files/332cdh/operatorsScore.csv | 1 + .../generated_files/332cdh/supportedExprs.csv | 2 + tools/generated_files/333/operatorsScore.csv | 1 + tools/generated_files/333/supportedExprs.csv | 2 + tools/generated_files/334/operatorsScore.csv | 1 + tools/generated_files/334/supportedExprs.csv | 2 + tools/generated_files/340/operatorsScore.csv | 1 + tools/generated_files/340/supportedExprs.csv | 2 + tools/generated_files/341/operatorsScore.csv | 1 + tools/generated_files/341/supportedExprs.csv | 2 + tools/generated_files/342/operatorsScore.csv | 1 + tools/generated_files/342/supportedExprs.csv | 2 + tools/generated_files/343/operatorsScore.csv | 1 + tools/generated_files/343/supportedExprs.csv | 2 + tools/generated_files/350/operatorsScore.csv | 1 + tools/generated_files/350/supportedExprs.csv | 2 + tools/generated_files/351/operatorsScore.csv | 1 + tools/generated_files/351/supportedExprs.csv | 2 + tools/generated_files/operatorsScore.csv | 1 + tools/generated_files/supportedExprs.csv | 2 + 71 files changed, 1514 insertions(+), 878 deletions(-) create mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala rename sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/{spark311/GpuBucketingUtils.scala => BucketingUtilsShim.scala} (87%) create mode 100644 sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala rename sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/{spark330/GpuBucketingUtils.scala => BucketingUtilsShim.scala} (85%) create mode 100644 sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala create mode 100644 tests/src/test/scala/com/nvidia/spark/rapids/HiveHashTestSuite.scala diff --git a/docs/additional-functionality/advanced_configs.md b/docs/additional-functionality/advanced_configs.md index f5d511cbbc5..2f76ffd8a68 100644 --- a/docs/additional-functionality/advanced_configs.md +++ b/docs/additional-functionality/advanced_configs.md @@ -266,6 +266,7 @@ Name | SQL Function(s) | Description | Default Value | Notes spark.rapids.sql.expression.GreaterThan|`>`|> operator|true|None| spark.rapids.sql.expression.GreaterThanOrEqual|`>=`|>= operator|true|None| spark.rapids.sql.expression.Greatest|`greatest`|Returns the greatest value of all parameters, skipping null values|true|None| +spark.rapids.sql.expression.HiveHash| |hive hash operator|true|None| spark.rapids.sql.expression.Hour|`hour`|Returns the hour component of the string/timestamp|true|None| spark.rapids.sql.expression.Hypot|`hypot`|Pythagorean addition (Hypotenuse) of real numbers|true|None| spark.rapids.sql.expression.If|`if`|IF expression|true|None| diff --git a/docs/supported_ops.md b/docs/supported_ops.md index fbafcfbf81d..018d6640976 100644 --- a/docs/supported_ops.md +++ b/docs/supported_ops.md @@ -7583,6 +7583,53 @@ are limited. NS +HiveHash + +hive hash operator +None +project +input +S +S +S +S +S +S +S +S +PS
UTC is only supported TZ for TIMESTAMP +S +NS +S +NS +NS +NS +NS +NS +NS + + +result + + + +S + + + + + + + + + + + + + + + + Hour `hour` Returns the hour component of the string/timestamp @@ -7855,6 +7902,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + InSet INSET operator @@ -7902,32 +7975,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - InitCap `initcap` Returns str with the first letter of each word in uppercase. All other letters are in lowercase @@ -8262,6 +8309,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + JsonToStructs `from_json` Returns a struct value with the given `jsonStr` and `schema` @@ -8309,32 +8382,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - JsonTuple `json_tuple` Returns a tuple like the function get_json_object, but it takes multiple names. All the input parameters and output column types are string. @@ -8654,6 +8701,32 @@ are limited. NS +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + LastDay `last_day` Returns the last day of the month which the date belongs to @@ -8701,32 +8774,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Lead `lead` Window function that returns N entries ahead of this one @@ -9042,6 +9089,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + LessThanOrEqual `<=` <= operator @@ -9174,32 +9247,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Like `like` Like @@ -9410,9 +9457,35 @@ are limited. -Log1p -`log1p` -Natural log 1 + expr +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + +Log1p +`log1p` +Natural log 1 + expr None project input @@ -9572,32 +9645,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Lower `lcase`, `lower` String lowercase operator @@ -9786,6 +9833,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + MapFilter `map_filter` Filters entries in a map using the function @@ -9948,32 +10021,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Md5 `md5` MD5 hash operator @@ -10162,6 +10209,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + MonotonicallyIncreasingID `monotonically_increasing_id` Returns monotonically increasing 64-bit integers @@ -10367,32 +10440,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Murmur3Hash `hash` Murmur3 hash operator @@ -10534,6 +10581,32 @@ are limited. NS +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Not `!`, `not` Boolean not operator @@ -10739,32 +10812,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Or `or` Logical OR @@ -10897,6 +10944,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + ParseUrl `parse_url` Extracts a part from a URL @@ -11101,32 +11174,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - PosExplode `posexplode_outer`, `posexplode` Given an input array produces a sequence of rows for each value in the array @@ -11306,6 +11353,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + PreciseTimestampConversion Expression used internally to convert the TimestampType to Long and back without losing precision, i.e. in microseconds. Used in time windowing @@ -11576,32 +11649,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Quarter `quarter` Returns the quarter of the year for date, in the range 1 to 4 @@ -11717,6 +11764,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + RaiseError `raise_error` Throw an exception @@ -11947,32 +12020,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - RegExpExtractAll `regexp_extract_all` Extract all strings matching a regular expression corresponding to the regex group index @@ -12172,6 +12219,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Remainder `%`, `mod` Remainder or modulo @@ -12334,32 +12407,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Rint `rint` Rounds up a double value to the nearest double equal to an integer @@ -12544,6 +12591,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + ScalaUDF User Defined Function, the UDF can choose to implement a RAPIDS accelerated interface to get better performance. @@ -12774,32 +12847,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - ShiftLeft `shiftleft` Bitwise shift left (<<) @@ -12936,6 +12983,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + ShiftRightUnsigned `shiftrightunsigned` Bitwise unsigned shift right (>>>) @@ -13141,32 +13214,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Sinh `sinh` Hyperbolic sine @@ -13304,6 +13351,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + SortArray `sort_array` Returns a sorted array with the input array and the ascending / descending order @@ -13489,54 +13562,28 @@ are limited. - - - -result - -S -S -S -S -NS -NS - - - -NS - - -S - - - - - - -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT + + + +result + +S +S +S +S +NS +NS + + + +NS + + +S + + + + Sqrt @@ -13697,6 +13744,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + StartsWith Starts with @@ -13922,32 +13995,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - StringLocate `locate`, `position` Substring search operator @@ -14126,6 +14173,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + StringRepeat `repeat` StringRepeat operator that repeats the given strings with numbers of times given by repeatTimes @@ -14283,32 +14356,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - StringSplit `split` Splits `str` around occurrences that match `regex` @@ -14487,6 +14534,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + StringTranslate `translate` StringTranslate operator @@ -14644,32 +14717,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - StringTrimLeft `ltrim` StringTrimLeft operator @@ -14853,6 +14900,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Substring `substr`, `substring` Substring operator @@ -15031,32 +15104,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Subtract `-` Subtraction @@ -15279,6 +15326,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Tanh `tanh` Hyperbolic tangent @@ -15437,32 +15510,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - ToDegrees `degrees` Converts radians to degrees @@ -15693,6 +15740,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + TransformKeys `transform_keys` Transform keys in a map using a transform function @@ -15829,32 +15902,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - UnaryMinus `negative` Negate a numeric value @@ -16061,6 +16108,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + UnboundedPreceding$ Special boundary for a window frame, indicating all rows preceding the current row @@ -16202,32 +16275,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Upper `ucase`, `upper` String uppercase operator @@ -16458,6 +16505,32 @@ are limited. NS +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + XxHash64 `xxhash64` xxhash64 hash operator @@ -16748,32 +16821,6 @@ are limited. S -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - ApproximatePercentile `approx_percentile`, `percentile_approx` Approximate percentile @@ -16948,6 +16995,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Average `avg`, `mean` Average aggregate operator @@ -17190,54 +17263,28 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -NS - - -result - - - - - - - - - - - - - - -PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT - - - - - -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT +NS + + +result + + + + + + + + + + + + + + +PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT + + + CollectSet @@ -17373,6 +17420,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Count `count` Count aggregate operator @@ -17639,32 +17712,6 @@ are limited. NS -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Last `last_value`, `last` last aggregate operator @@ -17798,6 +17845,32 @@ are limited. NS +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Max `max` Max aggregate operator @@ -18064,32 +18137,6 @@ are limited. NS -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - Percentile `percentile` Aggregation computing exact percentile @@ -18264,6 +18311,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + PivotFirst PivotFirst operator @@ -18529,32 +18602,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - StddevSamp `std`, `stddev_samp`, `stddev` Aggregation computing sample standard deviation @@ -18688,6 +18735,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + Sum `sum` Sum aggregate operator @@ -18954,32 +19027,6 @@ are limited. -Expression -SQL Functions(s) -Description -Notes -Context -Param/Output -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT - - VarianceSamp `var_samp`, `variance` Aggregation computing sample variance @@ -19113,6 +19160,32 @@ are limited. +Expression +SQL Functions(s) +Description +Notes +Context +Param/Output +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT + + NormalizeNaNAndZero Normalize NaN and zero diff --git a/integration_tests/src/main/python/datasourcev2_write_test.py b/integration_tests/src/main/python/datasourcev2_write_test.py index 88b7fd43b20..27c2be481d3 100644 --- a/integration_tests/src/main/python/datasourcev2_write_test.py +++ b/integration_tests/src/main/python/datasourcev2_write_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,36 +11,83 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import pytest -from asserts import assert_gpu_fallback_collect -from data_gen import * +from asserts import assert_gpu_fallback_collect, assert_equal_with_local_sort +from data_gen import gen_df, decimal_gens from marks import * -from pyspark.sql.types import * -from spark_session import is_hive_available, is_spark_330_or_later, with_cpu_session +from spark_session import is_hive_available, is_spark_330_or_later, with_cpu_session, with_gpu_session +from hive_parquet_write_test import _hive_bucket_gens, _hive_array_gens, _hive_struct_gens +from hive_parquet_write_test import read_single_bucket + +_hive_write_conf = { + "hive.enforce.bucketing": "true", + "hive.exec.dynamic.partition": "true", + "hive.exec.dynamic.partition.mode": "nonstrict"} + + +@pytest.mark.skipif(not (is_hive_available() and is_spark_330_or_later()), + reason="Must have Hive on Spark 3.3+") +@pytest.mark.parametrize('file_format', ['parquet', 'orc']) +def test_write_hive_bucketed_table(spark_tmp_table_factory, file_format): + num_rows = 2048 + + def gen_table(spark): + gen_list = [('_c' + str(i), gen) for i, gen in enumerate(_hive_bucket_gens)] + types_sql_str = ','.join('{} {}'.format( + name, gen.data_type.simpleString()) for name, gen in gen_list) + col_names_str = ','.join(name for name, gen in gen_list) + data_table = spark_tmp_table_factory.get() + gen_df(spark, gen_list, num_rows).createOrReplaceTempView(data_table) + return data_table, types_sql_str, col_names_str + + (input_data, input_schema, input_cols_str) = with_cpu_session(gen_table) + num_buckets = 4 + + def write_hive_table(spark, out_table): + spark.sql( + "create table {0} ({1}) stored as {2} clustered by ({3}) into {4} buckets".format( + out_table, input_schema, file_format, input_cols_str, num_buckets)) + spark.sql( + "insert into {0} select * from {1}".format(out_table, input_data)) + + cpu_table = spark_tmp_table_factory.get() + gpu_table = spark_tmp_table_factory.get() + with_cpu_session(lambda spark: write_hive_table(spark, cpu_table), _hive_write_conf) + with_gpu_session(lambda spark: write_hive_table(spark, gpu_table), _hive_write_conf) + cpu_rows, gpu_rows = 0, 0 + for cur_bucket_id in range(num_buckets): + # Verify the result bucket by bucket + ret_cpu = read_single_bucket(cpu_table, cur_bucket_id) + cpu_rows += len(ret_cpu) + ret_gpu = read_single_bucket(gpu_table, cur_bucket_id) + gpu_rows += len(ret_gpu) + assert_equal_with_local_sort(ret_cpu, ret_gpu) + + assert cpu_rows == num_rows + assert gpu_rows == num_rows + @ignore_order -@allow_non_gpu('DataWritingCommandExec,ExecutedCommandExec,WriteFilesExec') -@pytest.mark.skipif(not (is_hive_available() and is_spark_330_or_later()), reason="Must have Hive on Spark 3.3+") -@pytest.mark.parametrize('fileFormat', ['parquet', 'orc']) -def test_write_hive_bucketed_table_fallback(spark_tmp_table_factory, fileFormat): - """ - fallback because GPU does not support Hive hash partition - """ - table = spark_tmp_table_factory.get() +@allow_non_gpu('DataWritingCommandExec,ExecutedCommandExec,SortExec,WriteFilesExec') +@pytest.mark.skipif(not (is_hive_available() and is_spark_330_or_later()), + reason="Must have Hive on Spark 3.3+") +@pytest.mark.parametrize('file_format', ['parquet', 'orc']) +@pytest.mark.parametrize('gen', decimal_gens + _hive_array_gens + _hive_struct_gens) +def test_write_hive_bucketed_unsupported_types_fallback(spark_tmp_table_factory, file_format, gen): + out_table = spark_tmp_table_factory.get() def create_hive_table(spark): - spark.sql("""create table {0} (a bigint, b bigint, c bigint) - stored as {1} - clustered by (b) into 3 buckets""".format(table, fileFormat)) - return None - - conf = {"hive.enforce.bucketing": "true", - "hive.exec.dynamic.partition": "true", - "hive.exec.dynamic.partition.mode": "nonstrict"} - with_cpu_session(create_hive_table, conf = conf) + spark.sql("create table {0} (a {1}) stored as {2} clustered by (a) into 3 buckets".format( + out_table, gen.data_type.simpleString(), file_format)) + data_table = spark_tmp_table_factory.get() + gen_df(spark, [('a', gen)], length=10).createOrReplaceTempView(data_table) + return data_table + input_table = with_cpu_session(create_hive_table, _hive_write_conf) assert_gpu_fallback_collect( - lambda spark: spark.sql("insert into {} values (1, 2, 3)".format(table)), + lambda spark: spark.sql( + "insert into {0} select * from {1}".format(out_table, input_table)), 'DataWritingCommandExec', - conf = conf) + _hive_write_conf) diff --git a/integration_tests/src/main/python/hive_parquet_write_test.py b/integration_tests/src/main/python/hive_parquet_write_test.py index f62439a39af..d824bf7f4a0 100644 --- a/integration_tests/src/main/python/hive_parquet_write_test.py +++ b/integration_tests/src/main/python/hive_parquet_write_test.py @@ -14,20 +14,22 @@ import pytest -from asserts import assert_gpu_and_cpu_sql_writes_are_equal_collect +from asserts import assert_gpu_and_cpu_sql_writes_are_equal_collect, assert_equal_with_local_sort from conftest import is_databricks_runtime from data_gen import * from hive_write_test import _restricted_timestamp from marks import allow_non_gpu, ignore_order -from spark_session import with_cpu_session, is_before_spark_320, is_spark_350_or_later +from spark_session import with_cpu_session, with_gpu_session, is_before_spark_320, is_spark_350_or_later, is_before_spark_330, is_spark_330_or_later, is_databricks122_or_later # Disable the meta conversion from Hive write to FrameData write in Spark, to test # "GpuInsertIntoHiveTable" for Parquet write. _write_to_hive_conf = {"spark.sql.hive.convertMetastoreParquet": False} -_hive_basic_gens = [ - byte_gen, short_gen, int_gen, long_gen, float_gen, double_gen, string_gen, boolean_gen, - DateGen(start=date(1590, 1, 1)), _restricted_timestamp(), +_hive_bucket_gens = [ + boolean_gen, byte_gen, short_gen, int_gen, long_gen, string_gen, float_gen, double_gen, + DateGen(start=date(1590, 1, 1)), _restricted_timestamp()] + +_hive_basic_gens = _hive_bucket_gens + [ DecimalGen(precision=19, scale=1, nullable=True), DecimalGen(precision=23, scale=5, nullable=True), DecimalGen(precision=36, scale=3, nullable=True)] @@ -58,6 +60,20 @@ fallback_nodes = ['ProjectExec'] if is_databricks_runtime() or is_spark_350_or_later() else [] +def read_single_bucket(table, bucket_id): + # Bucket Id string format: f"_$id%05d" + ".c$fileCounter%03d". + # fileCounter is always 0 in this test. For example '_00002.c000' is for + # bucket id being 2. + # We leverage this bucket segment in the file path to filter rows belong to a bucket. + bucket_segment = '_' + "{}".format(bucket_id).rjust(5, '0') + '.c000' + return with_cpu_session( + lambda spark: spark.sql("select * from {}".format(table)) + .withColumn('file_name', f.input_file_name()) + .filter(f.col('file_name').contains(bucket_segment)) + .drop('file_name') # need to drop the file_name column for comparison. + .collect()) + + @allow_non_gpu(*(non_utc_allow + fallback_nodes)) @ignore_order(local=True) @pytest.mark.parametrize("is_ctas", [True, False], ids=['CTAS', 'CTTW']) @@ -174,3 +190,89 @@ def write_to_hive_sql(spark, output_table): spark_tmp_table_factory, write_to_hive_sql, _write_to_hive_conf) + + +@pytest.mark.skipif(is_before_spark_330() or (is_databricks_runtime() and not is_databricks122_or_later()), + reason="InsertIntoHiveTable supports bucketed write since Spark 330") +def test_insert_hive_bucketed_table(spark_tmp_table_factory): + num_rows = 2048 + + def gen_table(spark): + gen_list = [('_c' + str(i), gen) for i, gen in enumerate(_hive_bucket_gens)] + types_sql_str = ','.join('{} {}'.format( + name, gen.data_type.simpleString()) for name, gen in gen_list) + col_names_str = ','.join(name for name, gen in gen_list) + data_table = spark_tmp_table_factory.get() + gen_df(spark, gen_list, num_rows).createOrReplaceTempView(data_table) + return data_table, types_sql_str, col_names_str + + (input_data, input_schema, input_cols_str) = with_cpu_session(gen_table) + num_buckets = 4 + + def insert_hive_table(spark, out_table): + spark.sql( + "CREATE TABLE {} ({}) STORED AS PARQUET CLUSTERED BY ({}) INTO {} BUCKETS".format( + out_table, input_schema, input_cols_str, num_buckets)) + spark.sql( + "INSERT OVERWRITE {} SELECT * FROM {}".format(out_table, input_data)) + + cpu_table = spark_tmp_table_factory.get() + gpu_table = spark_tmp_table_factory.get() + with_cpu_session(lambda spark: insert_hive_table(spark, cpu_table), _write_to_hive_conf) + with_gpu_session(lambda spark: insert_hive_table(spark, gpu_table), _write_to_hive_conf) + cpu_rows, gpu_rows = 0, 0 + for cur_bucket_id in range(num_buckets): + # Verify the result bucket by bucket + ret_cpu = read_single_bucket(cpu_table, cur_bucket_id) + cpu_rows += len(ret_cpu) + ret_gpu = read_single_bucket(gpu_table, cur_bucket_id) + gpu_rows += len(ret_gpu) + assert_equal_with_local_sort(ret_cpu, ret_gpu) + + assert cpu_rows == num_rows + assert gpu_rows == num_rows + + +@pytest.mark.skipif(is_spark_330_or_later() or is_databricks122_or_later(), + reason="InsertIntoHiveTable supports bucketed write since Spark 330") +@pytest.mark.parametrize("hive_hash", [True, False]) +def test_insert_hive_bucketed_table_before_330(spark_tmp_table_factory, hive_hash): + num_buckets = 4 + + def insert_hive_table(spark, out_table): + data_table = spark_tmp_table_factory.get() + two_col_df(spark, int_gen, long_gen).createOrReplaceTempView(data_table) + spark.sql( + """CREATE TABLE {} (a int, b long) STORED AS PARQUET + CLUSTERED BY (a) INTO {} BUCKETS""".format(out_table, num_buckets)) + spark.sql( + "INSERT OVERWRITE {} SELECT * FROM {}".format(out_table, data_table)) + + all_confs = copy_and_update(_write_to_hive_conf, { + "hive.enforce.bucketing": False, # allow the write with bucket spec + "hive.enforce.sorting": False, # allow the write with bucket spec + "spark.rapids.sql.format.write.forceHiveHashForBucketing": hive_hash + }) + cpu_table = spark_tmp_table_factory.get() + gpu_table = spark_tmp_table_factory.get() + with_cpu_session(lambda spark: insert_hive_table(spark, cpu_table), all_confs) + with_gpu_session(lambda spark: insert_hive_table(spark, gpu_table), all_confs) + + all_cpu_rows = with_cpu_session( + lambda spark: spark.sql("select * from {}".format(cpu_table)).collect()) + all_gpu_rows = with_cpu_session( + lambda spark: spark.sql("select * from {}".format(gpu_table)).collect()) + assert_equal_with_local_sort(all_cpu_rows, all_gpu_rows) + + for cur_bucket_id in range(num_buckets): + ret_cpu = read_single_bucket(cpu_table, cur_bucket_id) + ret_gpu = read_single_bucket(gpu_table, cur_bucket_id) + if hive_hash: + # GPU will write the right bucketed table, but CPU will not. Because + # InsertIntoHiveTable supports bucketed write only since Spark 330. + # GPU behaviors differently than the normal Spark. + assert len(ret_gpu) > 0 and len(ret_cpu) == 0 + else: + # Both GPU and CPU write the data but no bucketing, actually. + assert len(ret_gpu) == 0 and len(ret_cpu) == 0 + diff --git a/integration_tests/src/main/python/hive_write_test.py b/integration_tests/src/main/python/hive_write_test.py index ae7052dffd7..945cc4806fb 100644 --- a/integration_tests/src/main/python/hive_write_test.py +++ b/integration_tests/src/main/python/hive_write_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -119,39 +119,21 @@ def test_optimized_hive_ctas_options_fallback(gens, storage_with_opts, spark_tmp spark_tmp_table_factory.get(), opts_string, storage, data_table)), fallback_class) -@allow_non_gpu('DataWritingCommandExec,ExecutedCommandExec,WriteFilesExec') -@pytest.mark.skipif(not (is_hive_available() and is_spark_33X() and not is_databricks122_or_later()), +@ignore_order +@pytest.mark.skipif(not (is_hive_available() and is_spark_330_or_later() and not is_databricks122_or_later()), reason="Requires Hive and Spark 3.3.X to write bucketed Hive tables") -@pytest.mark.parametrize("gens", [_basic_gens], ids=idfn) @pytest.mark.parametrize("storage", ["PARQUET", "ORC"], ids=idfn) -def test_optimized_hive_bucketed_fallback_33X(gens, storage, spark_tmp_table_factory): +def test_optimized_hive_ctas_bucketed_table(storage, spark_tmp_table_factory): in_table = spark_tmp_table_factory.get() - with_cpu_session(lambda spark: three_col_df(spark, int_gen, int_gen, int_gen).createOrReplaceTempView(in_table)) - assert_gpu_fallback_collect( - lambda spark: spark.sql( - """CREATE TABLE {} STORED AS {} - CLUSTERED BY (b) INTO 3 BUCKETS - AS SELECT * FROM {}""".format(spark_tmp_table_factory.get(), storage, in_table)), - "DataWritingCommandExec") - -# Since Spark 3.4.0, the internal "SortExec" will be pulled out by default -# from the FileFormatWriter. Then it is visible in the planning stage. -@allow_non_gpu("DataWritingCommandExec", "SortExec", "WriteFilesExec") -@pytest.mark.skipif(not (is_hive_available() and (is_spark_340_or_later() or is_databricks122_or_later())), - reason="Requires Hive and Spark 3.4+ to write bucketed Hive tables with SortExec pulled out") -@pytest.mark.parametrize("gens", [_basic_gens], ids=idfn) -@pytest.mark.parametrize("storage", ["PARQUET", "ORC"], ids=idfn) -@pytest.mark.parametrize("planned_write", [True, False], ids=idfn) -def test_optimized_hive_bucketed_fallback(gens, storage, planned_write, spark_tmp_table_factory): - in_table = spark_tmp_table_factory.get() - with_cpu_session(lambda spark: three_col_df(spark, int_gen, int_gen, int_gen).createOrReplaceTempView(in_table)) - assert_gpu_fallback_collect( - lambda spark: spark.sql( - """CREATE TABLE {} STORED AS {} - CLUSTERED BY (b) INTO 3 BUCKETS - AS SELECT * FROM {}""".format(spark_tmp_table_factory.get(), storage, in_table)), - "ExecutedCommandExec", - {"spark.sql.optimizer.plannedWrite.enabled": planned_write}) + # Supported types of Hive hash are all checked in datasourcev2_write_test, so here just + # verify the basic functionality by only the int_gen. + with_cpu_session(lambda spark: three_col_df( + spark, int_gen, int_gen, int_gen).createOrReplaceTempView(in_table)) + assert_gpu_and_cpu_sql_writes_are_equal_collect( + spark_tmp_table_factory, + lambda spark, out_table: """CREATE TABLE {} STORED AS {} + CLUSTERED BY (b) INTO 3 BUCKETS AS SELECT * FROM {}""".format( + out_table, storage, in_table)) def test_hive_copy_ints_to_long(spark_tmp_table_factory): do_hive_copy(spark_tmp_table_factory, int_gen, "INT", "BIGINT") diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 73475ef36f5..788ae4bde2f 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -323,8 +323,9 @@ final class InsertIntoHadoopFsRelationCommandMeta( private var fileFormat: Option[ColumnarFileFormat] = None override def tagSelfForGpuInternal(): Unit = { - if (GpuBucketingUtils.isHiveHashBucketing(cmd.options)) { - GpuBucketingUtils.tagForHiveBucketingWrite(this, cmd.bucketSpec, cmd.outputColumns, false) + if (BucketingUtilsShim.isHiveHashBucketing(cmd.options)) { + BucketingUtilsShim.tagForHiveBucketingWrite(this, cmd.bucketSpec, cmd.outputColumns, + conf.isForceHiveHashForBucketedWrite) } else { BucketIdMetaUtils.tagForBucketingWrite(this, cmd.bucketSpec, cmd.outputColumns) } @@ -3201,6 +3202,15 @@ object GpuOverrides extends Logging { def convertToGpu(): GpuExpression = GpuXxHash64(childExprs.map(_.convertToGpu()), a.seed) }), + expr[HiveHash]( + "hive hash operator", + ExprChecks.projectOnly(TypeSig.INT, TypeSig.INT, + repeatingParamCheck = Some(RepeatingParamCheck("input", + TypeSig.commonCudfTypes + TypeSig.NULL, TypeSig.all))), + (a, conf, p, r) => new ExprMeta[HiveHash](a, conf, p, r) { + def convertToGpu(): GpuExpression = + GpuHiveHash(childExprs.map(_.convertToGpu())) + }), expr[Contains]( "Contains", ExprChecks.binaryProject(TypeSig.BOOLEAN, TypeSig.BOOLEAN, diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index ba6ec25aeba..0a8ce614d83 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -1852,6 +1852,15 @@ val SHUFFLE_COMPRESSION_LZ4_CHUNK_SIZE = conf("spark.rapids.shuffle.compression. .bytesConf(ByteUnit.BYTE) .createWithDefault(64 * 1024) + val FORCE_HIVE_HASH_FOR_BUCKETED_WRITE = + conf("spark.rapids.sql.format.write.forceHiveHashForBucketing") + .doc("Hive write commands before Spark 330 use Murmur3Hash for bucketed write. " + + "When enabled, HiveHash will be always used for this instead of Murmur3. This is " + + "used to align with some customized Spark binaries before 330.") + .internal() + .booleanConf + .createWithDefault(false) + val SHUFFLE_MULTITHREADED_MAX_BYTES_IN_FLIGHT = conf("spark.rapids.shuffle.multiThreaded.maxBytesInFlight") .doc( @@ -3128,6 +3137,8 @@ class RapidsConf(conf: Map[String, String]) extends Logging { lazy val isFastSampleEnabled: Boolean = get(ENABLE_FAST_SAMPLE) + lazy val isForceHiveHashForBucketedWrite: Boolean = get(FORCE_HIVE_HASH_FOR_BUCKETED_WRITE) + lazy val isDetectDeltaLogQueries: Boolean = get(DETECT_DELTA_LOG_QUERIES) lazy val isDetectDeltaCheckpointQueries: Boolean = get(DETECT_DELTA_CHECKPOINT_QUERIES) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveFileFormat.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveFileFormat.scala index 69189b2600c..3b5244e5c79 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveFileFormat.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuHiveFileFormat.scala @@ -24,7 +24,7 @@ import com.google.common.base.Charsets import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.Arm.withResource import com.nvidia.spark.rapids.jni.CastStrings -import com.nvidia.spark.rapids.shims.GpuBucketingUtils +import com.nvidia.spark.rapids.shims.BucketingUtilsShim import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext} import org.apache.spark.internal.Logging @@ -44,8 +44,8 @@ object GpuHiveFileFormat extends Logging { def tagGpuSupport(meta: GpuInsertIntoHiveTableMeta): Option[ColumnarFileFormat] = { val insertCmd = meta.wrapped // Bucketing write - GpuBucketingUtils.tagForHiveBucketingWrite(meta, insertCmd.table.bucketSpec, - insertCmd.outputColumns, false) + BucketingUtilsShim.tagForHiveBucketingWrite(meta, insertCmd.table.bucketSpec, + insertCmd.outputColumns, meta.conf.isForceHiveHashForBucketedWrite) // Infer the file format from the serde string, similar as what Spark does in // RelationConversions for Hive. diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuSaveAsHiveFile.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuSaveAsHiveFile.scala index 667ad6dd4f4..08118cc11a0 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuSaveAsHiveFile.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/hive/rapids/GpuSaveAsHiveFile.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,6 +42,7 @@ private[hive] trait GpuSaveAsHiveFile extends GpuDataWritingCommand with SaveAsH hadoopConf: Configuration, fileFormat: ColumnarFileFormat, outputLocation: String, + forceHiveHashForBucketing: Boolean, customPartitionLocations: Map[TablePartitionSpec,String] = Map.empty, partitionAttributes: Seq[Attribute] = Nil, bucketSpec: Option[BucketSpec] = None, @@ -65,6 +66,7 @@ private[hive] trait GpuSaveAsHiveFile extends GpuDataWritingCommand with SaveAsH statsTrackers = Seq(gpuWriteJobStatsTracker(hadoopConf)), options = options, useStableSort = false, // TODO: Fetch from RapidsConf. + forceHiveHashForBucketing = forceHiveHashForBucketing, concurrentWriterPartitionFlushSize = 0L // TODO: Fetch from RapidsConf. ) } diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileFormatDataWriter.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileFormatDataWriter.scala index 939a421e0b9..be88e7a2937 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileFormatDataWriter.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileFormatDataWriter.scala @@ -35,7 +35,7 @@ import org.apache.spark.internal.io.FileCommitProtocol import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.catalog.{BucketSpec, ExternalCatalogUtils} import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec -import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, AttributeSet, Cast, Concat, Expression, Literal, Murmur3Hash, NullsFirst, ScalaUDF, UnsafeProjection} +import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, AttributeSet, Cast, Concat, Expression, HiveHash, Literal, Murmur3Hash, NullsFirst, ScalaUDF, UnsafeProjection} import org.apache.spark.sql.connector.write.DataWriter import org.apache.spark.sql.execution.datasources.{BucketingUtils, PartitioningUtils, WriteTaskResult} import org.apache.spark.sql.rapids.GpuFileFormatDataWriter._ @@ -945,9 +945,38 @@ object BucketIdMetaUtils { val hashMeta = GpuOverrides.wrapExpr(expr, meta.conf, None) hashMeta.tagForGpu() if(!hashMeta.canThisBeReplaced) { - meta.willNotWorkOnGpu(s"Hashing for generating bucket IDs can not run" + + meta.willNotWorkOnGpu(s"Murmur3 hashing for generating bucket IDs can not run" + s" on GPU. Details: ${hashMeta.explain(all=false)}") } } } + + def tagForBucketingHiveWrite(meta: RapidsMeta[_, _, _], bucketSpec: Option[BucketSpec], + outputColumns: Seq[Attribute]): Unit = { + bucketSpec.foreach { bSpec => + // Create a HiveHash expression to leverage the overriding types check. + val expr = HiveHash(bSpec.bucketColumnNames.map(n => outputColumns.find(_.name == n).get)) + val hashMeta = GpuOverrides.wrapExpr(expr, meta.conf, None) + hashMeta.tagForGpu() + if (!hashMeta.canThisBeReplaced) { + meta.willNotWorkOnGpu(s"Hive hashing for generating bucket IDs can not run" + + s" on GPU. Details: ${hashMeta.explain(all = false)}") + } + } + } + + def getWriteBucketSpecForHive( + bucketCols: Seq[Attribute], + numBuckets: Int): GpuWriterBucketSpec = { + // Hive bucketed table: use "HiveHash" and bitwise-and as bucket id expression. + // "bitwise-and" is used to handle the case of a wrong bucket id when + // the hash value is negative. + val hashId = GpuBitwiseAnd(GpuHiveHash(bucketCols), GpuLiteral(Int.MaxValue)) + val bucketIdExpression = GpuPmod(hashId, GpuLiteral(numBuckets)) + + // The bucket file name prefix is following Hive, Presto and Trino conversion, then + // Hive bucketed tables written by Plugin can be read by other SQL engines. + val fileNamePrefix = (bucketId: Int) => f"$bucketId%05d_0_" + GpuWriterBucketSpec(bucketIdExpression, fileNamePrefix) + } } diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala index ece5ef5acf5..2671323ea8c 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.rapids import java.io.IOException -import com.nvidia.spark.rapids.{ColumnarFileFormat, GpuDataWritingCommand} +import com.nvidia.spark.rapids.{ColumnarFileFormat, GpuDataWritingCommand, RapidsConf} import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.internal.io.FileCommitProtocol @@ -165,6 +165,8 @@ case class GpuInsertIntoHadoopFsRelationCommand( } } } + val forceHiveHashForBucketing = + RapidsConf.FORCE_HIVE_HASH_FOR_BUCKETED_WRITE.get(sparkSession.sessionState.conf) val updatedPartitionPaths = GpuFileFormatWriter.write( @@ -181,6 +183,7 @@ case class GpuInsertIntoHadoopFsRelationCommand( options = options, useStableSort = useStableSort, concurrentWriterPartitionFlushSize = concurrentWriterPartitionFlushSize, + forceHiveHashForBucketing = forceHiveHashForBucketing, numStaticPartitionCols = staticPartitions.size) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/HashFunctions.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/HashFunctions.scala index 36236b23936..bf568480baf 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/HashFunctions.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/HashFunctions.scala @@ -95,7 +95,6 @@ case class GpuMurmur3Hash(children: Seq[Expression], seed: Int) extends GpuHashE GpuColumnVector.from(GpuMurmur3Hash.compute(batch, children, seed), dataType) } - case class GpuXxHash64(children: Seq[Expression], seed: Long) extends GpuHashExpression { override def dataType: DataType = LongType @@ -108,3 +107,19 @@ case class GpuXxHash64(children: Seq[Expression], seed: Long) extends GpuHashExp } } } + +case class GpuHiveHash(children: Seq[Expression]) extends GpuHashExpression { + override def dataType: DataType = IntegerType + + override def prettyName: String = "hive-hash" + + override def columnarEval(batch: ColumnarBatch): GpuColumnVector = { + withResource(GpuProjectExec.project(batch, children)) { args => + val bases = GpuColumnVector.extractBases(args) + val normalized = bases.safeMap { cv => + HashUtils.normalizeInput(cv).asInstanceOf[ColumnView] + } + GpuColumnVector.from(withResource(normalized)(Hash.hiveHash), dataType) + } + } +} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala new file mode 100644 index 00000000000..40ea1d4145a --- /dev/null +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/*** spark-rapids-shim-json-lines +{"spark": "311"} +{"spark": "312"} +{"spark": "313"} +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable} + +object BucketSpecForHiveShim { + // Only for GpuInsertIntoHiveTable. The "InsertIntoHiveTable" in normal Spark before 330 + // does not execute the bucketed write even given a bucket specification. But some customized + // Spark binaries before 330 indeed will do it. So "forceHiveHash" is introduced to give a + // chance to enable the bucket write for this case. + def getBucketSpec(table: CatalogTable, forceHiveHash: Boolean): Option[BucketSpec] = { + if (forceHiveHash) { + table.bucketSpec + } else { + None + } + } +} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/spark311/GpuBucketingUtils.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BucketingUtilsShim.scala similarity index 87% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/spark311/GpuBucketingUtils.scala rename to sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BucketingUtilsShim.scala index a604267d1d9..835464ab88b 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/spark311/GpuBucketingUtils.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BucketingUtilsShim.scala @@ -32,7 +32,7 @@ import org.apache.spark.sql.catalyst.catalog.BucketSpec import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.rapids.{BucketIdMetaUtils, GpuWriterBucketSpec} -object GpuBucketingUtils { +object BucketingUtilsShim { def getWriterBucketSpec( bucketSpec: Option[BucketSpec], @@ -43,9 +43,7 @@ object GpuBucketingUtils { val bucketColumns = spec.bucketColumnNames.map(c => dataColumns.find(_.name == c).get) if (forceHiveHash) { // Forcely use HiveHash for Hive write commands for some customized Spark binaries. - // TODO: Cannot support this until we support Hive hash partitioning on the GPU - throw new UnsupportedOperationException("Hive hash partitioning is not supported" + - " on GPU") + BucketIdMetaUtils.getWriteBucketSpecForHive(bucketColumns, spec.numBuckets) } else { // Spark bucketed table: use `HashPartitioning.partitionIdExpression` as bucket id // expression, so that we can guarantee the data distribution is same between shuffle and @@ -67,9 +65,7 @@ object GpuBucketingUtils { def tagForHiveBucketingWrite(meta: RapidsMeta[_, _, _], bucketSpec: Option[BucketSpec], outColumns: Seq[Attribute], forceHiveHash: Boolean): Unit = { if (forceHiveHash) { - bucketSpec.foreach(_ => - meta.willNotWorkOnGpu("Hive Hashing for generating bucket IDs is not supported yet") - ) + BucketIdMetaUtils.tagForBucketingHiveWrite(meta, bucketSpec, outColumns) } else { BucketIdMetaUtils.tagForBucketingWrite(meta, bucketSpec, outColumns) } diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala index 55d9bc53704..ff9ca51b30a 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala +++ b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala @@ -184,8 +184,8 @@ final class OptimizedCreateHiveTableAsSelectCommandMeta( willNotWorkOnGpu("partitioned writes are not supported") } - GpuBucketingUtils.tagForHiveBucketingWrite(this, tableDesc.bucketSpec, - cmd.outputColumns, false) + BucketingUtilsShim.tagForHiveBucketingWrite(this, tableDesc.bucketSpec, + cmd.outputColumns, conf.isForceHiveHashForBucketedWrite) val serde = tableDesc.storage.serde.getOrElse("").toLowerCase(Locale.ROOT) if (serde.contains("parquet")) { diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala index acdd53b74ab..216235accf0 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala @@ -36,7 +36,7 @@ spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.hive.rapids.shims import com.nvidia.spark.rapids.{DataFromReplacementRule, DataWritingCommandMeta, GpuDataWritingCommand, GpuOverrides, RapidsConf, RapidsMeta} -import com.nvidia.spark.rapids.shims.{GpuBucketingUtils, GpuCreateHiveTableAsSelectBase} +import com.nvidia.spark.rapids.shims.{BucketingUtilsShim, GpuCreateHiveTableAsSelectBase} import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.catalyst.catalog.{CatalogTable, SessionCatalog} @@ -61,8 +61,8 @@ final class GpuCreateHiveTableAsSelectCommandMeta(cmd: CreateHiveTableAsSelectCo willNotWorkOnGpu("partitioned writes are not supported") } - GpuBucketingUtils.tagForHiveBucketingWrite(this, tableDesc.bucketSpec, - cmd.outputColumns, false) + BucketingUtilsShim.tagForHiveBucketingWrite(this, tableDesc.bucketSpec, + cmd.outputColumns, conf.isForceHiveHashForBucketedWrite) val catalog = spark.sessionState.catalog val tableExists = catalog.tableExists(tableDesc.identifier) diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala index 3f59d6565a5..08be08b9cd6 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala @@ -38,7 +38,7 @@ package org.apache.spark.sql.hive.rapids.shims import java.util.Locale import com.nvidia.spark.rapids.{ColumnarFileFormat, DataFromReplacementRule, DataWritingCommandMeta, GpuDataWritingCommand, RapidsConf, RapidsMeta} -import com.nvidia.spark.rapids.shims.GpuBucketingUtils +import com.nvidia.spark.rapids.shims.{BucketingUtilsShim, BucketSpecForHiveShim} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.hive.conf.HiveConf @@ -210,6 +210,8 @@ case class GpuInsertIntoHiveTable( // column names in order to make `loadDynamicPartitions` work. attr.withName(name.toLowerCase(Locale.ROOT)) } + val forceHiveHashForBucketing = + RapidsConf.FORCE_HIVE_HASH_FOR_BUCKETED_WRITE.get(sparkSession.sessionState.conf) val writtenParts = gpuSaveAsHiveFile( sparkSession = sparkSession, @@ -217,9 +219,10 @@ case class GpuInsertIntoHiveTable( hadoopConf = hadoopConf, fileFormat = fileFormat, outputLocation = tmpLocation.toString, + forceHiveHashForBucketing = forceHiveHashForBucketing, partitionAttributes = partitionAttributes, - bucketSpec = table.bucketSpec, - options = GpuBucketingUtils.getOptionsWithHiveBucketWrite(table.bucketSpec)) + bucketSpec = BucketSpecForHiveShim.getBucketSpec(table, forceHiveHashForBucketing), + options = BucketingUtilsShim.getOptionsWithHiveBucketWrite(table.bucketSpec)) if (partition.nonEmpty) { if (numDynamicPartitions > 0) { diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala index 4adbd7b2ef5..18446eca8a1 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala @@ -39,7 +39,7 @@ import java.util.{Date, UUID} import com.nvidia.spark.TimingUtils import com.nvidia.spark.rapids._ -import com.nvidia.spark.rapids.shims.{GpuBucketingUtils, RapidsFileSourceMetaUtils} +import com.nvidia.spark.rapids.shims.{BucketingUtilsShim, RapidsFileSourceMetaUtils} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.mapreduce._ @@ -106,6 +106,7 @@ object GpuFileFormatWriter extends Logging { options: Map[String, String], useStableSort: Boolean, concurrentWriterPartitionFlushSize: Long, + forceHiveHashForBucketing: Boolean = false, numStaticPartitionCols: Int = 0): Set[String] = { require(partitionColumns.size >= numStaticPartitionCols) @@ -136,8 +137,8 @@ object GpuFileFormatWriter extends Logging { if (projectList.nonEmpty) GpuProjectExec(projectList, plan)() else plan } - val writerBucketSpec = GpuBucketingUtils.getWriterBucketSpec(bucketSpec, dataColumns, - options, false) + val writerBucketSpec = BucketingUtilsShim.getWriterBucketSpec(bucketSpec, dataColumns, + options, forceHiveHashForBucketing) val sortColumns = bucketSpec.toSeq.flatMap { spec => spec.sortColumnNames.map(c => dataColumns.find(_.name == c).get) } diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala new file mode 100644 index 00000000000..119665a4131 --- /dev/null +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable} + +// This shim is used in GpuInsertIntoHiveTable for only Spark/cdh/db 33x. +object BucketSpecForHiveShim { + def getBucketSpec(table: CatalogTable, forceHiveHash: Boolean): Option[BucketSpec] = { + table.bucketSpec + } +} diff --git a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/GpuBucketingUtils.scala b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/BucketingUtilsShim.scala similarity index 85% rename from sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/GpuBucketingUtils.scala rename to sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/BucketingUtilsShim.scala index 0f7c9b4fd62..803bc0bc76a 100644 --- a/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/spark330/GpuBucketingUtils.scala +++ b/sql-plugin/src/main/spark330/scala/com/nvidia/spark/rapids/shims/BucketingUtilsShim.scala @@ -40,9 +40,9 @@ import com.nvidia.spark.rapids.RapidsMeta import org.apache.spark.sql.catalyst.catalog.BucketSpec import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.execution.datasources.BucketingUtils -import org.apache.spark.sql.rapids.GpuWriterBucketSpec +import org.apache.spark.sql.rapids.{BucketIdMetaUtils, GpuWriterBucketSpec} -object GpuBucketingUtils { +object BucketingUtilsShim { def getWriterBucketSpec( bucketSpec: Option[BucketSpec], @@ -54,9 +54,7 @@ object GpuBucketingUtils { val shouldHiveCompatibleWrite = options.getOrElse( BucketingUtils.optionForHiveCompatibleBucketWrite, "false").toBoolean if (shouldHiveCompatibleWrite) { - // TODO: Cannot support this until we support Hive hash partitioning on the GPU - throw new UnsupportedOperationException("Hive hash partitioning is not supported" + - " on GPU") + BucketIdMetaUtils.getWriteBucketSpecForHive(bucketColumns, spec.numBuckets) } else { // Spark bucketed table: use `HashPartitioning.partitionIdExpression` as bucket id // expression, so that we can guarantee the data distribution is same between shuffle and @@ -81,9 +79,7 @@ object GpuBucketingUtils { def tagForHiveBucketingWrite(meta: RapidsMeta[_, _, _], bucketSpec: Option[BucketSpec], outColumns: Seq[Attribute], forceHiveHash: Boolean): Unit = { - bucketSpec.foreach(_ => - // From Spark330, Hive write always uses HiveHash to generate bucket IDs. - meta.willNotWorkOnGpu("Hive Hashing for generating bucket IDs is not supported yet") - ) + // From Spark330, Hive write always uses HiveHash to generate bucket IDs. + BucketIdMetaUtils.tagForBucketingHiveWrite(meta, bucketSpec, outColumns) } } diff --git a/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala new file mode 100644 index 00000000000..dd4f01137a3 --- /dev/null +++ b/sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "330db"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable} + +object BucketSpecForHiveShim { + // Only for GpuInsertIntoHiveTable. On DB 330, the "InsertIntoHiveTable" does not + // execute the bucketed write even given a bucket specification. + def getBucketSpec(table: CatalogTable, forceHiveHash: Boolean): Option[BucketSpec] = { + None + } +} diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala index b3103c3c76e..18b768a0909 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala @@ -30,7 +30,7 @@ package org.apache.spark.sql.hive.rapids.shims import java.util.Locale import com.nvidia.spark.rapids.{ColumnarFileFormat, DataFromReplacementRule, DataWritingCommandMeta, GpuDataWritingCommand, RapidsConf, RapidsMeta} -import com.nvidia.spark.rapids.shims.GpuBucketingUtils +import com.nvidia.spark.rapids.shims.BucketingUtilsShim import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.hive.conf.HiveConf @@ -199,6 +199,9 @@ case class GpuInsertIntoHiveTable( // column names in order to make `loadDynamicPartitions` work. attr.withName(name.toLowerCase(Locale.ROOT)) } + val forceHiveHashForBucketing = + RapidsConf.FORCE_HIVE_HASH_FOR_BUCKETED_WRITE.get(sparkSession.sessionState.conf) + val writtenParts = gpuSaveAsHiveFile( sparkSession = sparkSession, @@ -206,9 +209,10 @@ case class GpuInsertIntoHiveTable( hadoopConf = hadoopConf, fileFormat = fileFormat, outputLocation = tmpLocation.toString, + forceHiveHashForBucketing = forceHiveHashForBucketing, partitionAttributes = partitionAttributes, bucketSpec = table.bucketSpec, - options = GpuBucketingUtils.getOptionsWithHiveBucketWrite(table.bucketSpec)) + options = BucketingUtilsShim.getOptionsWithHiveBucketWrite(table.bucketSpec)) if (partition.nonEmpty) { if (numDynamicPartitions > 0) { diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala index e74bf979af9..1a2389bffd3 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala @@ -199,7 +199,8 @@ final class OptimizedCreateHiveTableAsSelectCommandMeta( val outputColumns = DataWritingCommand.logicalPlanOutputWithNames(cmd.query, cmd.outputColumnNames) - GpuBucketingUtils.tagForHiveBucketingWrite(this, tableDesc.bucketSpec, outputColumns, false) + BucketingUtilsShim.tagForHiveBucketingWrite(this, tableDesc.bucketSpec, outputColumns, + conf.isForceHiveHashForBucketedWrite) val serde = tableDesc.storage.serde.getOrElse("").toLowerCase(Locale.ROOT) if (serde.contains("parquet")) { diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala index 874d89353aa..2241d6771f8 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala @@ -31,7 +31,7 @@ import java.util.{Date, UUID} import com.nvidia.spark.TimingUtils import com.nvidia.spark.rapids._ -import com.nvidia.spark.rapids.shims.{GpuBucketingUtils, RapidsFileSourceMetaUtils} +import com.nvidia.spark.rapids.shims.{BucketingUtilsShim, RapidsFileSourceMetaUtils} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.mapreduce._ @@ -99,6 +99,7 @@ object GpuFileFormatWriter extends Logging { options: Map[String, String], useStableSort: Boolean, concurrentWriterPartitionFlushSize: Long, + forceHiveHashForBucketing: Boolean = false, numStaticPartitionCols: Int = 0): Set[String] = { require(partitionColumns.size >= numStaticPartitionCols) @@ -119,8 +120,8 @@ object GpuFileFormatWriter extends Logging { .map(RapidsFileSourceMetaUtils.cleanupFileSourceMetadataInformation)) val dataColumns = finalOutputSpec.outputColumns.filterNot(partitionSet.contains) - val writerBucketSpec = GpuBucketingUtils.getWriterBucketSpec(bucketSpec, dataColumns, - options, false) + val writerBucketSpec = BucketingUtilsShim.getWriterBucketSpec(bucketSpec, dataColumns, + options, forceHiveHashForBucketing) val sortColumns = bucketSpec.toSeq.flatMap { spec => spec.sortColumnNames.map(c => dataColumns.find(_.name == c).get) } diff --git a/tests/src/test/scala/com/nvidia/spark/rapids/HiveHashTestSuite.scala b/tests/src/test/scala/com/nvidia/spark/rapids/HiveHashTestSuite.scala new file mode 100644 index 00000000000..d5883ac5420 --- /dev/null +++ b/tests/src/test/scala/com/nvidia/spark/rapids/HiveHashTestSuite.scala @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids + +import java.lang.{Byte => JByte} + +import ai.rapids.cudf.{ColumnVector => CudfColumnVector} +import com.nvidia.spark.rapids.Arm.{closeOnExcept, withResource} + +import org.apache.spark.sql.catalyst.expressions.{BoundReference, ExprId, HiveHash} +import org.apache.spark.sql.rapids.GpuHiveHash +import org.apache.spark.sql.types._ +import org.apache.spark.sql.vectorized.ColumnarBatch + +class HiveHashTestSuite extends SparkQueryCompareTestSuite { + // All columns should have the same length(now is 15) for multiple columns tests. + def genBoolColumn: GpuColumnVector = GpuColumnVector.from( + CudfColumnVector.fromBoxedBooleans(null, true, null, false, true, true, true, + false, false, null, false, true, null, false, true), + BooleanType) + + def genByteColumn: GpuColumnVector = { + val bytes: Seq[JByte] = Seq[JByte](null, Byte.MaxValue, Byte.MinValue, null, null) ++ + Seq(0, -0, 1, -1, 10, -10, 126, -126, 111, -111).map(b => JByte.valueOf(b.toByte)) + GpuColumnVector.from(CudfColumnVector.fromBoxedBytes(bytes: _*), ByteType) + } + + def genIntColumn: GpuColumnVector = GpuColumnVector.from( + CudfColumnVector.fromBoxedInts(null, Int.MaxValue, Int.MinValue, 0, -0, 1, -1, + null, 100, -100, null, null, 99, -99, null), + IntegerType) + + def genLongColumn: GpuColumnVector = GpuColumnVector.from( + CudfColumnVector.fromBoxedLongs(null, Long.MaxValue, Long.MinValue, 0, -0, 1, -1, + null, 100, -100, null, null, 99, -99, null), + LongType) + + def genStringColumn: GpuColumnVector = GpuColumnVector.from( + CudfColumnVector.fromStrings(null, "", "1", "432", "a\nb", "a", "`@@$*&", " ", "\t", + "dE\"\u0100\t\u0101 \ud720\ud721", "\ud720\ud721\ud720\ud721", "''", null, " ", + "This is a long string (greater than 128 bytes/char string) case to test this " + + "hash function. Just want an abnormal case here to see if any error may " + + "happen when doing the hive hashing"), + StringType) + + def genFloatColumn: GpuColumnVector = GpuColumnVector.from( + CudfColumnVector.fromBoxedFloats(null, 0.0f, -0.0f, 99.0f, -99.0f, Float.NaN, + Float.MaxValue, Float.MinValue, Float.MinPositiveValue, Float.NegativeInfinity, + Float.PositiveInfinity, + FLOAT_POSITIVE_NAN_LOWER_RANGE, FLOAT_POSITIVE_NAN_UPPER_RANGE, + FLOAT_NEGATIVE_NAN_LOWER_RANGE, FLOAT_NEGATIVE_NAN_UPPER_RANGE), + FloatType) + + def genDoubleColumn: GpuColumnVector = GpuColumnVector.from( + CudfColumnVector.fromBoxedDoubles(null, 0.0, -0.0, 199.0, -199.0, Double.NaN, + Double.MaxValue, Double.MinValue, Double.MinPositiveValue, Double.NegativeInfinity, + Double.PositiveInfinity, + DOUBLE_POSITIVE_NAN_LOWER_RANGE, DOUBLE_POSITIVE_NAN_UPPER_RANGE, + DOUBLE_NEGATIVE_NAN_LOWER_RANGE, DOUBLE_NEGATIVE_NAN_UPPER_RANGE), + DoubleType) + + def genDateColumn: GpuColumnVector = GpuColumnVector.from( + CudfColumnVector.timestampDaysFromBoxedInts(null, 0, null, 100, -100, 0x12345678, + null, -0x12345678, 171922, 19899, 17766, -0, 16897, null, 18888), + DateType) + + def genTimestampColumn: GpuColumnVector = GpuColumnVector.from( + CudfColumnVector.timestampMicroSecondsFromBoxedLongs(null, 0, -0, 100, -100, + 0x123456789abcdefL, -0x123456789abcdefL, 1719561256196L, -1719561256196L, null, + 1234567890, -1234567890, 999, -999, null), + TimestampType) + + private def testHiveHashOnGpuAndCpuThenClose(cols: GpuColumnVector*): Unit = { + val (numRows, cpuRefs, gpuRefs) = closeOnExcept(cols) { _ => + val rowsNum = cols.head.getRowCount + require(cols.tail.forall(_.getRowCount == rowsNum), + s"All the input columns should have the same length: $rowsNum.") + + val cpuRefs = cols.zipWithIndex.map { case (col, id) => + BoundReference(id, col.dataType(), nullable = true) + } + val gpuRefs = cols.zipWithIndex.map { case (col, id) => + GpuBoundReference(id, col.dataType(), nullable = true)(ExprId(id), + s"col${id}_${col.dataType().simpleString}") + } + (rowsNum.toInt, cpuRefs, gpuRefs) + } + + // GPU run + val gpuRet = closeOnExcept(cols) { _ => + val inputCB = new ColumnarBatch(cols.toArray, numRows) + val hostRet = withResource(GpuHiveHash(gpuRefs).columnarEval(inputCB)) { retCol => + retCol.copyToHost() + } + withResource(hostRet) { _ => + (0 until numRows).map(hostRet.getInt).toArray + } + } + // CPU run + val cpuRet = withResource(cols) { _ => + withResource(new ColumnarBatch(cols.map(_.copyToHost()).toArray, numRows)) { cb => + val hiveHash = HiveHash(cpuRefs) + val it = cb.rowIterator() + (0 until numRows).map(_ => hiveHash.eval(it.next())).toArray + } + } + assertResult(cpuRet)(gpuRet) + } + + test("Test hive hash booleans") { + testHiveHashOnGpuAndCpuThenClose(genBoolColumn) + } + + test("Test hive hash bytes") { + testHiveHashOnGpuAndCpuThenClose(genByteColumn) + } + + test("Test hive hash ints") { + testHiveHashOnGpuAndCpuThenClose(genIntColumn) + } + + test("Test hive hash longs") { + testHiveHashOnGpuAndCpuThenClose(genLongColumn) + } + + test("Test hive hash floats") { + testHiveHashOnGpuAndCpuThenClose(genFloatColumn) + } + + test("Test hive hash doubles") { + testHiveHashOnGpuAndCpuThenClose(genDoubleColumn) + } + + test("Test hive hash dates") { + testHiveHashOnGpuAndCpuThenClose(genDateColumn) + } + + test("Test hive hash timestamps") { + testHiveHashOnGpuAndCpuThenClose(genTimestampColumn) + } + + test("Test hive hash strings") { + testHiveHashOnGpuAndCpuThenClose(genStringColumn) + } + + test("Test hive hash mixed {bytes, ints, longs, dates}") { + val cols = closeOnExcept(new Array[GpuColumnVector](4)) { buf => + buf(0) = genByteColumn + buf(1) = genIntColumn + buf(2) = genLongColumn + buf(3) = genDateColumn + buf + } + testHiveHashOnGpuAndCpuThenClose(cols: _*) + } + + test("Test hive hash mixed {booleans, floats, doubles, strings}") { + val cols = closeOnExcept(new Array[GpuColumnVector](4)) { buf => + buf(0) = genBoolColumn + buf(1) = genFloatColumn + buf(2) = genDoubleColumn + buf(3) = genStringColumn + buf + } + testHiveHashOnGpuAndCpuThenClose(cols: _*) + } +} diff --git a/tools/generated_files/311/operatorsScore.csv b/tools/generated_files/311/operatorsScore.csv index e3f8d1053c1..c5c2080694c 100644 --- a/tools/generated_files/311/operatorsScore.csv +++ b/tools/generated_files/311/operatorsScore.csv @@ -126,6 +126,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/311/supportedExprs.csv b/tools/generated_files/311/supportedExprs.csv index 5f57725522f..88f6ee0d6d3 100644 --- a/tools/generated_files/311/supportedExprs.csv +++ b/tools/generated_files/311/supportedExprs.csv @@ -254,6 +254,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/312/operatorsScore.csv b/tools/generated_files/312/operatorsScore.csv index e3f8d1053c1..c5c2080694c 100644 --- a/tools/generated_files/312/operatorsScore.csv +++ b/tools/generated_files/312/operatorsScore.csv @@ -126,6 +126,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/312/supportedExprs.csv b/tools/generated_files/312/supportedExprs.csv index 5f57725522f..88f6ee0d6d3 100644 --- a/tools/generated_files/312/supportedExprs.csv +++ b/tools/generated_files/312/supportedExprs.csv @@ -254,6 +254,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/313/operatorsScore.csv b/tools/generated_files/313/operatorsScore.csv index e3f8d1053c1..c5c2080694c 100644 --- a/tools/generated_files/313/operatorsScore.csv +++ b/tools/generated_files/313/operatorsScore.csv @@ -126,6 +126,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/313/supportedExprs.csv b/tools/generated_files/313/supportedExprs.csv index 5f57725522f..88f6ee0d6d3 100644 --- a/tools/generated_files/313/supportedExprs.csv +++ b/tools/generated_files/313/supportedExprs.csv @@ -254,6 +254,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/320/operatorsScore.csv b/tools/generated_files/320/operatorsScore.csv index a9606fe77d9..0c7295f95ac 100644 --- a/tools/generated_files/320/operatorsScore.csv +++ b/tools/generated_files/320/operatorsScore.csv @@ -130,6 +130,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/320/supportedExprs.csv b/tools/generated_files/320/supportedExprs.csv index 937ed7ae569..f9e47f10110 100644 --- a/tools/generated_files/320/supportedExprs.csv +++ b/tools/generated_files/320/supportedExprs.csv @@ -254,6 +254,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/321/operatorsScore.csv b/tools/generated_files/321/operatorsScore.csv index a9606fe77d9..0c7295f95ac 100644 --- a/tools/generated_files/321/operatorsScore.csv +++ b/tools/generated_files/321/operatorsScore.csv @@ -130,6 +130,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/321/supportedExprs.csv b/tools/generated_files/321/supportedExprs.csv index 937ed7ae569..f9e47f10110 100644 --- a/tools/generated_files/321/supportedExprs.csv +++ b/tools/generated_files/321/supportedExprs.csv @@ -254,6 +254,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/321cdh/operatorsScore.csv b/tools/generated_files/321cdh/operatorsScore.csv index a9606fe77d9..0c7295f95ac 100644 --- a/tools/generated_files/321cdh/operatorsScore.csv +++ b/tools/generated_files/321cdh/operatorsScore.csv @@ -130,6 +130,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/321cdh/supportedExprs.csv b/tools/generated_files/321cdh/supportedExprs.csv index 937ed7ae569..f9e47f10110 100644 --- a/tools/generated_files/321cdh/supportedExprs.csv +++ b/tools/generated_files/321cdh/supportedExprs.csv @@ -254,6 +254,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/322/operatorsScore.csv b/tools/generated_files/322/operatorsScore.csv index a9606fe77d9..0c7295f95ac 100644 --- a/tools/generated_files/322/operatorsScore.csv +++ b/tools/generated_files/322/operatorsScore.csv @@ -130,6 +130,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/322/supportedExprs.csv b/tools/generated_files/322/supportedExprs.csv index 937ed7ae569..f9e47f10110 100644 --- a/tools/generated_files/322/supportedExprs.csv +++ b/tools/generated_files/322/supportedExprs.csv @@ -254,6 +254,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/323/operatorsScore.csv b/tools/generated_files/323/operatorsScore.csv index a9606fe77d9..0c7295f95ac 100644 --- a/tools/generated_files/323/operatorsScore.csv +++ b/tools/generated_files/323/operatorsScore.csv @@ -130,6 +130,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/323/supportedExprs.csv b/tools/generated_files/323/supportedExprs.csv index 937ed7ae569..f9e47f10110 100644 --- a/tools/generated_files/323/supportedExprs.csv +++ b/tools/generated_files/323/supportedExprs.csv @@ -254,6 +254,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/324/operatorsScore.csv b/tools/generated_files/324/operatorsScore.csv index a9606fe77d9..0c7295f95ac 100644 --- a/tools/generated_files/324/operatorsScore.csv +++ b/tools/generated_files/324/operatorsScore.csv @@ -130,6 +130,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/324/supportedExprs.csv b/tools/generated_files/324/supportedExprs.csv index 937ed7ae569..f9e47f10110 100644 --- a/tools/generated_files/324/supportedExprs.csv +++ b/tools/generated_files/324/supportedExprs.csv @@ -254,6 +254,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/330/operatorsScore.csv b/tools/generated_files/330/operatorsScore.csv index 7de435ebbc7..235432d36e1 100644 --- a/tools/generated_files/330/operatorsScore.csv +++ b/tools/generated_files/330/operatorsScore.csv @@ -135,6 +135,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/330/supportedExprs.csv b/tools/generated_files/330/supportedExprs.csv index ce504a2ca68..30719957123 100644 --- a/tools/generated_files/330/supportedExprs.csv +++ b/tools/generated_files/330/supportedExprs.csv @@ -263,6 +263,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/330cdh/operatorsScore.csv b/tools/generated_files/330cdh/operatorsScore.csv index 7de435ebbc7..235432d36e1 100644 --- a/tools/generated_files/330cdh/operatorsScore.csv +++ b/tools/generated_files/330cdh/operatorsScore.csv @@ -135,6 +135,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/330cdh/supportedExprs.csv b/tools/generated_files/330cdh/supportedExprs.csv index ce504a2ca68..30719957123 100644 --- a/tools/generated_files/330cdh/supportedExprs.csv +++ b/tools/generated_files/330cdh/supportedExprs.csv @@ -263,6 +263,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/331/operatorsScore.csv b/tools/generated_files/331/operatorsScore.csv index b1168d0fa6b..6b4cd0349c6 100644 --- a/tools/generated_files/331/operatorsScore.csv +++ b/tools/generated_files/331/operatorsScore.csv @@ -136,6 +136,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/331/supportedExprs.csv b/tools/generated_files/331/supportedExprs.csv index 44a7a8b977c..0d02f7e5f42 100644 --- a/tools/generated_files/331/supportedExprs.csv +++ b/tools/generated_files/331/supportedExprs.csv @@ -265,6 +265,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/332/operatorsScore.csv b/tools/generated_files/332/operatorsScore.csv index b1168d0fa6b..6b4cd0349c6 100644 --- a/tools/generated_files/332/operatorsScore.csv +++ b/tools/generated_files/332/operatorsScore.csv @@ -136,6 +136,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/332/supportedExprs.csv b/tools/generated_files/332/supportedExprs.csv index 44a7a8b977c..0d02f7e5f42 100644 --- a/tools/generated_files/332/supportedExprs.csv +++ b/tools/generated_files/332/supportedExprs.csv @@ -265,6 +265,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/332cdh/operatorsScore.csv b/tools/generated_files/332cdh/operatorsScore.csv index b1168d0fa6b..6b4cd0349c6 100644 --- a/tools/generated_files/332cdh/operatorsScore.csv +++ b/tools/generated_files/332cdh/operatorsScore.csv @@ -136,6 +136,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/332cdh/supportedExprs.csv b/tools/generated_files/332cdh/supportedExprs.csv index 44a7a8b977c..0d02f7e5f42 100644 --- a/tools/generated_files/332cdh/supportedExprs.csv +++ b/tools/generated_files/332cdh/supportedExprs.csv @@ -265,6 +265,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/333/operatorsScore.csv b/tools/generated_files/333/operatorsScore.csv index b1168d0fa6b..6b4cd0349c6 100644 --- a/tools/generated_files/333/operatorsScore.csv +++ b/tools/generated_files/333/operatorsScore.csv @@ -136,6 +136,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/333/supportedExprs.csv b/tools/generated_files/333/supportedExprs.csv index 44a7a8b977c..0d02f7e5f42 100644 --- a/tools/generated_files/333/supportedExprs.csv +++ b/tools/generated_files/333/supportedExprs.csv @@ -265,6 +265,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/334/operatorsScore.csv b/tools/generated_files/334/operatorsScore.csv index b1168d0fa6b..6b4cd0349c6 100644 --- a/tools/generated_files/334/operatorsScore.csv +++ b/tools/generated_files/334/operatorsScore.csv @@ -136,6 +136,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/334/supportedExprs.csv b/tools/generated_files/334/supportedExprs.csv index 44a7a8b977c..0d02f7e5f42 100644 --- a/tools/generated_files/334/supportedExprs.csv +++ b/tools/generated_files/334/supportedExprs.csv @@ -265,6 +265,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/340/operatorsScore.csv b/tools/generated_files/340/operatorsScore.csv index 161fcc90e7b..575e3ba3617 100644 --- a/tools/generated_files/340/operatorsScore.csv +++ b/tools/generated_files/340/operatorsScore.csv @@ -137,6 +137,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/340/supportedExprs.csv b/tools/generated_files/340/supportedExprs.csv index 63bfecc4ce3..e6533b50427 100644 --- a/tools/generated_files/340/supportedExprs.csv +++ b/tools/generated_files/340/supportedExprs.csv @@ -265,6 +265,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/341/operatorsScore.csv b/tools/generated_files/341/operatorsScore.csv index 161fcc90e7b..575e3ba3617 100644 --- a/tools/generated_files/341/operatorsScore.csv +++ b/tools/generated_files/341/operatorsScore.csv @@ -137,6 +137,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/341/supportedExprs.csv b/tools/generated_files/341/supportedExprs.csv index 63bfecc4ce3..e6533b50427 100644 --- a/tools/generated_files/341/supportedExprs.csv +++ b/tools/generated_files/341/supportedExprs.csv @@ -265,6 +265,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/342/operatorsScore.csv b/tools/generated_files/342/operatorsScore.csv index 161fcc90e7b..575e3ba3617 100644 --- a/tools/generated_files/342/operatorsScore.csv +++ b/tools/generated_files/342/operatorsScore.csv @@ -137,6 +137,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/342/supportedExprs.csv b/tools/generated_files/342/supportedExprs.csv index 63bfecc4ce3..e6533b50427 100644 --- a/tools/generated_files/342/supportedExprs.csv +++ b/tools/generated_files/342/supportedExprs.csv @@ -265,6 +265,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/343/operatorsScore.csv b/tools/generated_files/343/operatorsScore.csv index 161fcc90e7b..575e3ba3617 100644 --- a/tools/generated_files/343/operatorsScore.csv +++ b/tools/generated_files/343/operatorsScore.csv @@ -137,6 +137,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/343/supportedExprs.csv b/tools/generated_files/343/supportedExprs.csv index 63bfecc4ce3..e6533b50427 100644 --- a/tools/generated_files/343/supportedExprs.csv +++ b/tools/generated_files/343/supportedExprs.csv @@ -265,6 +265,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/350/operatorsScore.csv b/tools/generated_files/350/operatorsScore.csv index d98c632ac68..17b19e9a810 100644 --- a/tools/generated_files/350/operatorsScore.csv +++ b/tools/generated_files/350/operatorsScore.csv @@ -138,6 +138,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/350/supportedExprs.csv b/tools/generated_files/350/supportedExprs.csv index 6c34e85c530..c658185a47e 100644 --- a/tools/generated_files/350/supportedExprs.csv +++ b/tools/generated_files/350/supportedExprs.csv @@ -265,6 +265,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/351/operatorsScore.csv b/tools/generated_files/351/operatorsScore.csv index d98c632ac68..17b19e9a810 100644 --- a/tools/generated_files/351/operatorsScore.csv +++ b/tools/generated_files/351/operatorsScore.csv @@ -138,6 +138,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/351/supportedExprs.csv b/tools/generated_files/351/supportedExprs.csv index 6c34e85c530..c658185a47e 100644 --- a/tools/generated_files/351/supportedExprs.csv +++ b/tools/generated_files/351/supportedExprs.csv @@ -265,6 +265,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/operatorsScore.csv b/tools/generated_files/operatorsScore.csv index e3f8d1053c1..c5c2080694c 100644 --- a/tools/generated_files/operatorsScore.csv +++ b/tools/generated_files/operatorsScore.csv @@ -126,6 +126,7 @@ GreaterThan,4 GreaterThanOrEqual,4 Greatest,4 HiveGenericUDF,4 +HiveHash,4 HiveSimpleUDF,4 Hour,4 Hypot,4 diff --git a/tools/generated_files/supportedExprs.csv b/tools/generated_files/supportedExprs.csv index 5f57725522f..88f6ee0d6d3 100644 --- a/tools/generated_files/supportedExprs.csv +++ b/tools/generated_files/supportedExprs.csv @@ -254,6 +254,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA From c49d69326b433592ff7b10942f334706b5d85161 Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Fri, 5 Jul 2024 08:05:08 -0700 Subject: [PATCH 65/79] Handle the change for UnaryPositive now extending RuntimeReplaceable [databricks] (#11137) * Handle the change for UnaryPositive now extending RuntimeReplaceable * signing off Signed-off-by: Raza Jafri * Revert "Handle the change for UnaryPositive now extending RuntimeReplaceable" This reverts commit 414db71ce732157f26101fcd6e7ec0bb808ebc9b. * Replace UnaryExprMeta with ExprMeta for UnaryPositive * Replace UnaryExprMeta with ExprMeta for UnaryPositive * override isFoldableNonLitAllowed --------- Signed-off-by: Raza Jafri --- .../main/scala/com/nvidia/spark/rapids/GpuOverrides.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 788ae4bde2f..3ec8d2c1d08 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -1148,8 +1148,11 @@ object GpuOverrides extends Logging { TypeSig.astTypes + GpuTypeShims.additionalArithmeticSupportedTypes, TypeSig.gpuNumeric + GpuTypeShims.additionalArithmeticSupportedTypes, TypeSig.numericAndInterval), - (a, conf, p, r) => new UnaryAstExprMeta[UnaryPositive](a, conf, p, r) { - override def convertToGpu(child: Expression): GpuExpression = GpuUnaryPositive(child) + (a, conf, p, r) => new ExprMeta[UnaryPositive](a, conf, p, r) { + override val isFoldableNonLitAllowed: Boolean = true + override def convertToGpu(): GpuExpression = + GpuUnaryPositive(childExprs.head.convertToGpu()) + }), expr[Year]( "Returns the year from a date or timestamp", From 78f0403dd4ee6acfc78c9bf6c1ad72fe2438e50f Mon Sep 17 00:00:00 2001 From: Peixin Date: Mon, 8 Jul 2024 09:12:57 +0800 Subject: [PATCH 66/79] Update fastparquet to 2024.5.0 for numpy2 compatibility [databricks] (#11138) * update fastparquet version to 2024.5.0 for numpy2 compatibility Signed-off-by: Peixin Li * Revert "WAR numpy2 failed fastparquet compatibility issue (#11072)" This reverts commit 6eb854dc82a4ec0e96b97871a97c6b9ed7471722. * update databricks pip install command * include py3.8 compatibility * copyright --------- Signed-off-by: Peixin Li --- integration_tests/requirements.txt | 5 +++-- .../src/main/python/fastparquet_compatibility_test.py | 4 +--- jenkins/Dockerfile-blossom.integration.rocky | 2 +- jenkins/Dockerfile-blossom.integration.ubuntu | 4 ++-- jenkins/Dockerfile-blossom.ubuntu | 2 +- jenkins/databricks/setup.sh | 6 +++--- 6 files changed, 11 insertions(+), 12 deletions(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 570d01f5119..602f6b4943b 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,4 +17,5 @@ pandas pyarrow pytest-xdist >= 2.0.0 findspark -fastparquet == 0.8.3 \ No newline at end of file +fastparquet == 0.8.3 ; python_version == '3.8' +fastparquet == 2024.5.0 ; python_version >= '3.9' diff --git a/integration_tests/src/main/python/fastparquet_compatibility_test.py b/integration_tests/src/main/python/fastparquet_compatibility_test.py index 4b0fc2827f4..bdfb7d6e35c 100644 --- a/integration_tests/src/main/python/fastparquet_compatibility_test.py +++ b/integration_tests/src/main/python/fastparquet_compatibility_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -30,8 +30,6 @@ def fastparquet_unavailable(): return False except ImportError: return True - except ValueError: # TODO: remove when https://github.com/NVIDIA/spark-rapids/issues/11070 is fixed - return True rebase_write_corrected_conf = { diff --git a/jenkins/Dockerfile-blossom.integration.rocky b/jenkins/Dockerfile-blossom.integration.rocky index 749d29ae58c..e3d83adf85e 100644 --- a/jenkins/Dockerfile-blossom.integration.rocky +++ b/jenkins/Dockerfile-blossom.integration.rocky @@ -57,7 +57,7 @@ RUN export CUDA_VER=`echo ${CUDA_VER} | cut -d '.' -f 1,2` && \ conda install -y -c conda-forge sre_yield && \ conda clean -ay # install pytest plugins for xdist parallel run -RUN python -m pip install findspark pytest-xdist pytest-order fastparquet==0.8.3 +RUN python -m pip install findspark pytest-xdist pytest-order fastparquet==2024.5.0 # Set default java as 1.8.0 ENV JAVA_HOME "/usr/lib/jvm/java-1.8.0-openjdk" diff --git a/jenkins/Dockerfile-blossom.integration.ubuntu b/jenkins/Dockerfile-blossom.integration.ubuntu index 1dd81ca68ed..6ac7a514361 100644 --- a/jenkins/Dockerfile-blossom.integration.ubuntu +++ b/jenkins/Dockerfile-blossom.integration.ubuntu @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -69,7 +69,7 @@ RUN export CUDA_VER=`echo ${CUDA_VER} | cut -d '.' -f 1,2` && \ conda install -y -c conda-forge sre_yield && \ conda clean -ay # install pytest plugins for xdist parallel run -RUN python -m pip install findspark pytest-xdist pytest-order fastparquet==0.8.3 +RUN python -m pip install findspark pytest-xdist pytest-order fastparquet==2024.5.0 RUN apt install -y inetutils-ping expect diff --git a/jenkins/Dockerfile-blossom.ubuntu b/jenkins/Dockerfile-blossom.ubuntu index 755dd3eeaa0..7cb2122af26 100644 --- a/jenkins/Dockerfile-blossom.ubuntu +++ b/jenkins/Dockerfile-blossom.ubuntu @@ -61,7 +61,7 @@ RUN update-java-alternatives --set /usr/lib/jvm/java-1.8.0-openjdk-${ARCH} RUN ln -sfn /usr/bin/python3.9 /usr/bin/python RUN ln -sfn /usr/bin/python3.9 /usr/bin/python3 -RUN python -m pip install pytest sre_yield requests pandas pyarrow findspark pytest-xdist pre-commit pytest-order fastparquet==0.8.3 +RUN python -m pip install pytest sre_yield requests pandas pyarrow findspark pytest-xdist pre-commit pytest-order fastparquet==2024.5.0 RUN UCX_CUDA_VER=`echo ${CUDA_VER} | cut -d '.' -f1` && \ mkdir -p /tmp/ucx && \ diff --git a/jenkins/databricks/setup.sh b/jenkins/databricks/setup.sh index 63e71d3a25d..b0c5c5c65bd 100755 --- a/jenkins/databricks/setup.sh +++ b/jenkins/databricks/setup.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -52,5 +52,5 @@ PYTHON_SITE_PACKAGES="$HOME/.local/lib/${PYTHON_VERSION}/site-packages" $PYSPARK_PYTHON -m pip install --target $PYTHON_SITE_PACKAGES pytest sre_yield requests pandas pyarrow findspark pytest-xdist pytest-order # Install fastparquet (and numpy as its dependency). -$PYSPARK_PYTHON -m pip install --target $PYTHON_SITE_PACKAGES numpy -$PYSPARK_PYTHON -m pip install --target $PYTHON_SITE_PACKAGES fastparquet==0.8.3 +echo -e "fastparquet==0.8.3;python_version=='3.8'\nfastparquet==2024.5.0;python_version>='3.9'" > fastparquet.txt +$PYSPARK_PYTHON -m pip install --target $PYTHON_SITE_PACKAGES -r fastparquet.txt From 7894d51b8954e8fab1bd6671e81c2fee7023ad3a Mon Sep 17 00:00:00 2001 From: Zach Puller Date: Mon, 8 Jul 2024 06:44:21 -0700 Subject: [PATCH 67/79] upgrade ucx to 1.17.0 (#11147) Signed-off-by: Zach Puller --- .../shuffle-docker-examples/Dockerfile.rocky_no_rdma | 2 +- .../shuffle-docker-examples/Dockerfile.rocky_rdma | 2 +- .../shuffle-docker-examples/Dockerfile.ubuntu_no_rdma | 2 +- .../shuffle-docker-examples/Dockerfile.ubuntu_rdma | 2 +- jenkins/Dockerfile-blossom.ubuntu | 4 ++-- pom.xml | 2 +- scala2.13/pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/additional-functionality/shuffle-docker-examples/Dockerfile.rocky_no_rdma b/docs/additional-functionality/shuffle-docker-examples/Dockerfile.rocky_no_rdma index fe5c64b1dfc..b78a25feed7 100644 --- a/docs/additional-functionality/shuffle-docker-examples/Dockerfile.rocky_no_rdma +++ b/docs/additional-functionality/shuffle-docker-examples/Dockerfile.rocky_no_rdma @@ -24,7 +24,7 @@ # - ROCKY_VER: Rocky Linux OS version ARG CUDA_VER=11.8.0 -ARG UCX_VER=1.16.0 +ARG UCX_VER=1.17.0 ARG UCX_CUDA_VER=11 ARG UCX_ARCH=x86_64 ARG ROCKY_VER=8 diff --git a/docs/additional-functionality/shuffle-docker-examples/Dockerfile.rocky_rdma b/docs/additional-functionality/shuffle-docker-examples/Dockerfile.rocky_rdma index f88c4212a92..7e15218bf5a 100644 --- a/docs/additional-functionality/shuffle-docker-examples/Dockerfile.rocky_rdma +++ b/docs/additional-functionality/shuffle-docker-examples/Dockerfile.rocky_rdma @@ -24,7 +24,7 @@ # - ROCKY_VER: Rocky Linux OS version ARG CUDA_VER=11.8.0 -ARG UCX_VER=1.16.0 +ARG UCX_VER=1.17.0 ARG UCX_CUDA_VER=11 ARG UCX_ARCH=x86_64 ARG ROCKY_VER=8 diff --git a/docs/additional-functionality/shuffle-docker-examples/Dockerfile.ubuntu_no_rdma b/docs/additional-functionality/shuffle-docker-examples/Dockerfile.ubuntu_no_rdma index 792e7848e56..8bb01f7b1ff 100644 --- a/docs/additional-functionality/shuffle-docker-examples/Dockerfile.ubuntu_no_rdma +++ b/docs/additional-functionality/shuffle-docker-examples/Dockerfile.ubuntu_no_rdma @@ -25,7 +25,7 @@ # ARG CUDA_VER=11.8.0 -ARG UCX_VER=1.16.0 +ARG UCX_VER=1.17.0 ARG UCX_CUDA_VER=11 ARG UCX_ARCH=x86_64 ARG UBUNTU_VER=20.04 diff --git a/docs/additional-functionality/shuffle-docker-examples/Dockerfile.ubuntu_rdma b/docs/additional-functionality/shuffle-docker-examples/Dockerfile.ubuntu_rdma index 42014c67251..044e4b7411c 100644 --- a/docs/additional-functionality/shuffle-docker-examples/Dockerfile.ubuntu_rdma +++ b/docs/additional-functionality/shuffle-docker-examples/Dockerfile.ubuntu_rdma @@ -35,7 +35,7 @@ ARG RDMA_CORE_VERSION=32.1 ARG CUDA_VER=11.8.0 -ARG UCX_VER=1.16.0 +ARG UCX_VER=1.17.0 ARG UCX_CUDA_VER=11 ARG UCX_ARCH=x86_64 ARG UBUNTU_VER=20.04 diff --git a/jenkins/Dockerfile-blossom.ubuntu b/jenkins/Dockerfile-blossom.ubuntu index 7cb2122af26..c5802b122af 100644 --- a/jenkins/Dockerfile-blossom.ubuntu +++ b/jenkins/Dockerfile-blossom.ubuntu @@ -21,7 +21,7 @@ # Arguments: # CUDA_VER=[11.X.Y,12.X.Y] # UBUNTU_VER=[20.04,22.0.4] -# UCX_VER=1.16.0 +# UCX_VER=1.17.0 # TARGETPLATFORM=[linux/amd64,linux/arm64] # ARCH=[amd64,arm64] # UCX_ARCH=[x86_64,aarch64] @@ -29,7 +29,7 @@ ARG CUDA_VER=11.8.0 ARG UBUNTU_VER=20.04 -ARG UCX_VER=1.16.0 +ARG UCX_VER=1.17.0 ARG TARGETPLATFORM=linux/amd64 # multi-platform build with: docker buildx build --platform linux/arm64,linux/amd64 on either amd64 or arm64 host # check available official arm-based docker images at https://hub.docker.com/r/nvidia/cuda/tags (OS/ARCH) diff --git a/pom.xml b/pom.xml index 3ff87c3cb97..5fb949ca34d 100644 --- a/pom.xml +++ b/pom.xml @@ -732,7 +732,7 @@ https://github.com/openjdk/jdk17/blob/4afbcaf55383ec2f5da53282a1547bac3d099e9d/src/jdk.compiler/share/classes/com/sun/tools/javac/resources/compiler.properties#L1993-L1994 --> -Xlint:all,-serial,-path,-try,-processing|-Werror - 1.16.0 + 1.17.0 1.0.6 ${ucx.baseVersion} diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index e32a64f0529..3b6080eb56c 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -732,7 +732,7 @@ https://github.com/openjdk/jdk17/blob/4afbcaf55383ec2f5da53282a1547bac3d099e9d/src/jdk.compiler/share/classes/com/sun/tools/javac/resources/compiler.properties#L1993-L1994 --> -Xlint:all,-serial,-path,-try,-processing|-Werror - 1.16.0 + 1.17.0 1.0.6 ${ucx.baseVersion} From d8041882e11efa3f235661d15a8974f7d9663e54 Mon Sep 17 00:00:00 2001 From: Liangcai Li Date: Mon, 8 Jul 2024 23:27:58 +0800 Subject: [PATCH 68/79] Fix the test error of bucketed write for non-utc (#11151) Signed-off-by: Firestarman --- integration_tests/src/main/python/hive_parquet_write_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/src/main/python/hive_parquet_write_test.py b/integration_tests/src/main/python/hive_parquet_write_test.py index d824bf7f4a0..7dc10d64e49 100644 --- a/integration_tests/src/main/python/hive_parquet_write_test.py +++ b/integration_tests/src/main/python/hive_parquet_write_test.py @@ -192,6 +192,7 @@ def write_to_hive_sql(spark, output_table): _write_to_hive_conf) +@allow_non_gpu(*non_utc_allow) @pytest.mark.skipif(is_before_spark_330() or (is_databricks_runtime() and not is_databricks122_or_later()), reason="InsertIntoHiveTable supports bucketed write since Spark 330") def test_insert_hive_bucketed_table(spark_tmp_table_factory): From 18babedb40ec9243fdf671f22c149b1f082996dd Mon Sep 17 00:00:00 2001 From: MithunR Date: Mon, 8 Jul 2024 14:30:54 -0700 Subject: [PATCH 69/79] Fix ANSI mode failures in subquery_test.py [databricks] (#11102) * Fix ANSI mode failures in subquery_test.py. Fixes #11029. Some tests in subquery_test.py fail when run with ANSI mode enabled, because certain array columns are accessed with invalid indices. These tests predate the availability of ANSI mode in Spark. This commit modifies the tests so that the generated data is appropriately sized for the query. There is no loss of test coverage; failure cases for invalid index values in array columns are already tested as part of `array_test::test_array_item_ansi_fail_invalid_index` Signed-off-by: MithunR --- .../src/main/python/subquery_test.py | 189 ++++++++++++------ 1 file changed, 126 insertions(+), 63 deletions(-) diff --git a/integration_tests/src/main/python/subquery_test.py b/integration_tests/src/main/python/subquery_test.py index e6d641d4212..5fb73ff2367 100644 --- a/integration_tests/src/main/python/subquery_test.py +++ b/integration_tests/src/main/python/subquery_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,122 +13,185 @@ # limitations under the License. import pytest -from asserts import assert_gpu_and_cpu_are_equal_sql +from asserts import assert_gpu_and_cpu_are_equal_sql, assert_gpu_and_cpu_error +from conftest import spark_tmp_table_factory from data_gen import * from marks import * @ignore_order(local=True) @pytest.mark.parametrize('data_gen', all_basic_gens, ids=idfn) -def test_scalar_subquery_basics(data_gen): +def test_scalar_subquery_basics(spark_tmp_table_factory, data_gen): # Fix num_slices at 1 to make sure that first/last returns same results under CPU and GPU. + table_name = spark_tmp_table_factory.get() assert_gpu_and_cpu_are_equal_sql( lambda spark: gen_df(spark, [('a', data_gen)], num_slices=1), - 'table', - '''select a, (select last(a) from table) - from table - where a > (select first(a) from table) + table_name, + f'''select a, (select last(a) from {table_name}) + from {table_name} + where a > (select first(a) from {table_name}) ''') + @ignore_order(local=True) @pytest.mark.parametrize('basic_gen', all_basic_gens, ids=idfn) -def test_scalar_subquery_struct(basic_gen): +def test_scalar_subquery_struct(spark_tmp_table_factory, basic_gen): # single-level struct gen = [('ss', StructGen([['a', basic_gen], ['b', basic_gen]]))] + table_name = spark_tmp_table_factory.get() assert_gpu_and_cpu_are_equal_sql( # Fix num_slices at 1 to make sure that first/last returns same results under CPU and GPU. lambda spark: gen_df(spark, gen, num_slices=1), - 'table', - '''select ss, (select last(ss) from table) - from table - where (select first(ss) from table).b > ss.a + table_name, + f'''select ss, (select last(ss) from {table_name}) + from {table_name} + where (select first(ss) from {table_name}).b > ss.a ''') + # nested struct gen = [('ss', StructGen([['child', StructGen([['c0', basic_gen]])]]))] assert_gpu_and_cpu_are_equal_sql( # Fix num_slices at 1 to make sure that first/last returns same results under CPU and GPU. lambda spark: gen_df(spark, gen, num_slices=1), - 'table', - '''select ss, (select last(ss) from table) - from table - where (select first(ss) from table)['child']['c0'] > ss.child.c0 + table_name, + f'''select ss, (select last(ss) from {table_name}) + from {table_name} + where (select first(ss) from {table_name})['child']['c0'] > ss.child.c0 ''') + # struct of array - gen = [('ss', StructGen([['arr', ArrayGen(basic_gen)]]))] + # Note: The test query accesses the first two elements of the array. The datagen is set up + # to generate arrays of a minimum of two elements. Otherwise, the test will fail in ANSI mode. + # No meaningful test coverage is lost. Accessing invalid indices of arrays is already tested + # as part of array_test.py::test_array_item_ansi_fail_invalid_index. + gen = [('ss', StructGen([['arr', ArrayGen(basic_gen, min_length=2)]]))] assert_gpu_and_cpu_are_equal_sql( # Fix num_slices at 1 to make sure that first/last returns same results under CPU and GPU. lambda spark: gen_df(spark, gen, length=100, num_slices=1), - 'table', - '''select sort_array(ss.arr), sort_array((select last(ss) from table)['arr']) - from table - where (select first(ss) from table).arr[0] > ss.arr[1] + table_name, + f'''select sort_array(ss.arr), sort_array((select last(ss) from {table_name})['arr']) + from {table_name} + where (select first(ss) from {table_name}).arr[0] > ss.arr[1] ''') + @ignore_order(local=True) +@pytest.mark.parametrize('is_ansi_enabled', [False, True]) @pytest.mark.parametrize('basic_gen', all_basic_gens, ids=idfn) -def test_scalar_subquery_array(basic_gen): +def test_scalar_subquery_array(spark_tmp_table_factory, is_ansi_enabled, basic_gen): + """ + For this test, all the array inputs are sized so that ArrayIndexOutOfBounds conditions are + avoided. This is to ensure that the tests don't fail with exceptions in ANSI mode. + Note that no meaningful test coverage is lost here. ArrayIndexOutOfBounds exceptions are + already tested as part of array_test.py::test_array_item_ansi_fail_invalid_index. + """ + conf = {'spark.sql.ansi.enabled': is_ansi_enabled} + table_name = spark_tmp_table_factory.get() + # single-level array + test_array_gen = ArrayGen(basic_gen, min_length=1 if is_ansi_enabled else 0) assert_gpu_and_cpu_are_equal_sql( # Fix num_slices at 1 to make sure that first/last returns same results under CPU and GPU. - lambda spark: gen_df(spark, [('arr', ArrayGen(basic_gen))], num_slices=1), - 'table', - '''select sort_array(arr), - sort_array((select last(arr) from table)) - from table - where (select first(arr) from table)[0] > arr[0] - ''') + lambda spark: gen_df(spark, [('arr', test_array_gen)], num_slices=1), + table_name, + f'''select sort_array(arr), + sort_array((select last(arr) from {table_name})) + from {table_name} + where (select first(arr) from {table_name})[0] > arr[0] + ''', + conf=conf) + # nested array + test_array_gen = ArrayGen(ArrayGen(basic_gen, min_length=2 if is_ansi_enabled else 0), + min_length=11 if is_ansi_enabled else 0) assert_gpu_and_cpu_are_equal_sql( # Fix num_slices at 1 to make sure that first/last returns same results under CPU and GPU. - lambda spark: gen_df(spark, [('arr', ArrayGen(ArrayGen(basic_gen)))] - , length=100 - , num_slices=1), - 'table', - '''select sort_array(arr[10]), - sort_array((select last(arr) from table)[10]) - from table - where (select first(arr) from table)[0][1] > arr[0][1] - ''') + lambda spark: gen_df(spark, [('arr', test_array_gen)], length=100, num_slices=1), + table_name, + f'''select sort_array(arr[10]), + sort_array((select last(arr) from {table_name})[10]) + from {table_name} + where (select first(arr) from {table_name})[0][1] > arr[0][1] + ''', + conf=conf) + # array of struct + test_array_gen = ArrayGen(StructGen([['a', basic_gen]]), min_length=11 if is_ansi_enabled else 0) assert_gpu_and_cpu_are_equal_sql( # Fix num_slices at 1 to make sure that first/last returns same results under CPU and GPU. - lambda spark: gen_df(spark, [('arr', ArrayGen(StructGen([['a', basic_gen]])))] - , length=100 - , num_slices=1), - 'table', - '''select arr[10].a, (select last(arr) from table)[10].a - from table - where (select first(arr) from table)[0].a > arr[0].a - ''') + lambda spark: gen_df(spark, [('arr', test_array_gen)], length=100, num_slices=1), + table_name, + f'''select arr[10].a, (select last(arr) from {table_name})[10].a + from {table_name} + where (select first(arr) from {table_name})[0].a > arr[0].a + ''', + conf=conf) + + +@ignore_order(local=True) +def test_scalar_subquery_array_ansi_mode_failures(spark_tmp_table_factory): + """ + This tests the case where the array scalar returned from a subquery might be indexed into + with an out-of-range index value. With ANSI mode enabled, an exception is expected. + + A more thorough test for invalid indices is done in array_test.py::test_array_item_ansi_fail_invalid_index, + and is out of the scope of this test. + """ + table_name = spark_tmp_table_factory.get() + + def test_function(spark): + # Fix num_slices at 1 to make sure that first/last returns same results under CPU and GPU. + df = gen_df(spark, [('arr', ArrayGen(long_gen))], num_slices=1) + df.createOrReplaceTempView(table_name) + query = f''' + SELECT SORT_ARRAY(arr), + SORT_ARRAY((SELECT LAST(arr) FROM {table_name})) + FROM {table_name} + WHERE (SELECT CAST(ARRAY() AS ARRAY))[0] > arr[0] + ''' + return spark.sql(query) + + assert_gpu_and_cpu_error( + lambda spark: test_function(spark).collect(), + conf=ansi_enabled_conf, + error_message='ArrayIndexOutOfBoundsException') + @ignore_order(local=True) -def test_scalar_subquery_map(): +def test_scalar_subquery_map(spark_tmp_table_factory): + # Note: For this test, all the array inputs are sized so that ArrayIndexOutOfBounds conditions are + # avoided. This is to ensure that the tests don't fail with exceptions in ANSI mode. + # Note that no meaningful test coverage is lost here. ArrayIndexOutOfBounds exceptions are + # already tested as part of array_test.py::test_array_item_ansi_fail_invalid_index. + table_name = spark_tmp_table_factory.get() map_gen = map_string_string_gen[0] assert_gpu_and_cpu_are_equal_sql( # Fix num_slices at 1 to make sure that first/last returns same results under CPU and GPU. lambda spark: gen_df(spark, [('kv', map_gen)], length=100, num_slices=1), - 'table', - '''select kv['key_0'], - (select first(kv) from table)['key_1'], - (select last(kv) from table)['key_2'] - from table + table_name, + f'''select kv['key_0'], + (select first(kv) from {table_name})['key_1'], + (select last(kv) from {table_name})['key_2'] + from {table_name} ''') + # array of map assert_gpu_and_cpu_are_equal_sql( # Fix num_slices at 1 to make sure that first/last returns same results under CPU and GPU. - lambda spark: gen_df(spark, [('arr', ArrayGen(map_gen))], length=100, num_slices=1), - 'table', - '''select arr[0]['key_0'], - (select first(arr) from table)[0]['key_1'], - (select last(arr[0]) from table)['key_2'] - from table + lambda spark: gen_df(spark, [('arr', ArrayGen(map_gen, min_length=1))], length=100, num_slices=1), + table_name, + f'''select arr[0]['key_0'], + (select first(arr) from {table_name})[0]['key_1'], + (select last(arr[0]) from {table_name})['key_2'] + from {table_name} ''') + # struct of map assert_gpu_and_cpu_are_equal_sql( # Fix num_slices at 1 to make sure that first/last returns same results under CPU and GPU. lambda spark: gen_df(spark, [('ss', StructGen([['kv', map_gen]]))], length=100, num_slices=1), - 'table', - '''select ss['kv']['key_0'], - (select first(ss) from table)['kv']['key_1'], - (select last(ss.kv) from table)['key_2'] - from table + table_name, + f'''select ss['kv']['key_0'], + (select first(ss) from {table_name})['kv']['key_1'], + (select last(ss.kv) from {table_name})['key_2'] + from {table_name} ''') From a056f162bf1fe21d8e3ce58b646bc791f419210a Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Tue, 9 Jul 2024 09:45:43 +0800 Subject: [PATCH 70/79] Fix LORE dump oom. (#11153) * Fix oom * Remove unused code Signed-off-by: liurenjie1024 * Update copy right --------- Signed-off-by: liurenjie1024 --- .../com/nvidia/spark/rapids/DumpUtils.scala | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/DumpUtils.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/DumpUtils.scala index 21d2de6ad68..64b08162557 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/DumpUtils.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/DumpUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -149,30 +149,24 @@ object DumpUtils extends Logging { class ParquetDumper(private val outputStream: OutputStream, table: Table) extends HostBufferConsumer with AutoCloseable { private[this] val tempBuffer = new Array[Byte](128 * 1024) - private[this] val buffers = mutable.Queue[(HostMemoryBuffer, Long)]() def this(path: String, table: Table) = { this(new FileOutputStream(path), table) } - val tableWriter: TableWriter = { + private lazy val tableWriter: TableWriter = { // avoid anything conversion, just dump as it is val builder = ParquetDumper.parquetWriterOptionsFromTable(ParquetWriterOptions.builder(), table) .withCompressionType(ParquetDumper.COMPRESS_TYPE) Table.writeParquetChunked(builder.build(), this) } - override - def handleBuffer(buffer: HostMemoryBuffer, len: Long): Unit = - buffers += Tuple2(buffer, len) - - def writeBufferedData(): Unit = { - ColumnarOutputWriter.writeBufferedData(buffers, tempBuffer, outputStream) - } + override def handleBuffer(buffer: HostMemoryBuffer, len: Long): Unit = + ColumnarOutputWriter.writeBufferedData(mutable.Queue((buffer, len)), tempBuffer, + outputStream) def writeTable(table: Table): Unit = { tableWriter.write(table) - writeBufferedData() } /** @@ -181,7 +175,6 @@ class ParquetDumper(private val outputStream: OutputStream, table: Table) extend */ def close(): Unit = { tableWriter.close() - writeBufferedData() outputStream.close() } } From 6f36d354917bc249ec05844afb7e28532240ac94 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Tue, 9 Jul 2024 09:09:06 -0500 Subject: [PATCH 71/79] Fix batch splitting for partition column size on row-count-only batches (#11156) Signed-off-by: Jason Lowe --- integration_tests/src/main/python/parquet_test.py | 11 +++++++++++ .../nvidia/spark/rapids/BatchWithPartitionData.scala | 9 ++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/integration_tests/src/main/python/parquet_test.py b/integration_tests/src/main/python/parquet_test.py index 7928f6a5a5b..e21ba622f46 100644 --- a/integration_tests/src/main/python/parquet_test.py +++ b/integration_tests/src/main/python/parquet_test.py @@ -1492,3 +1492,14 @@ def test_parquet_column_name_with_dots(spark_tmp_path, reader_confs): assert_gpu_and_cpu_are_equal_collect(lambda spark: reader(spark).selectExpr("`a.b`"), conf=all_confs) assert_gpu_and_cpu_are_equal_collect(lambda spark: reader(spark).selectExpr("`a.b`.`c.d.e`.`f.g`"), conf=all_confs) + +def test_parquet_partition_batch_row_count_only_splitting(spark_tmp_path): + data_path = spark_tmp_path + "/PARQUET_DATA" + def setup_table(spark): + spark.range(1000).withColumn("p", f.lit("x")).coalesce(1)\ + .write\ + .partitionBy("p")\ + .parquet(data_path) + with_cpu_session(lambda spark: setup_table(spark)) + assert_gpu_and_cpu_are_equal_collect(lambda spark: spark.read.parquet(data_path).select("p"), + conf={"spark.rapids.sql.columnSizeBytes": "100"}) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/BatchWithPartitionData.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/BatchWithPartitionData.scala index f6429ddd709..d0f61d884d7 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/BatchWithPartitionData.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/BatchWithPartitionData.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -413,6 +413,13 @@ object BatchWithPartitionDataUtils { if (splitIndices.isEmpty) { Seq(SpillableColumnarBatch(GpuColumnVector.incRefCounts(input), SpillPriorities.ACTIVE_ON_DECK_PRIORITY)) + } else if (input.numCols() == 0) { + // calculate the number of rows for each batch based on the split indices + val batchRows = + splitIndices.head +: splitIndices.zip(splitIndices.tail :+ input.numRows()).map { + case (startRow, endRow) => endRow - startRow + } + batchRows.map(numRows => new JustRowsColumnarBatch(numRows)) } else { val schema = GpuColumnVector.extractTypes(input) val splitInput = withResource(GpuColumnVector.from(input)) { table => From 29904a3ee37e23316ea47f7995840a4993b6690b Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Tue, 9 Jul 2024 22:21:49 +0800 Subject: [PATCH 72/79] Add deletion vector metrics for low shuffle merge. (#11132) * Add deletion vector metrics * Add for databricks Signed-off-by: liurenjie1024 * Fix comments * Fix comments --------- Signed-off-by: liurenjie1024 --- .../GpuDeltaParquetFileFormatUtils.scala | 53 +++++++++++-------- .../GpuDelta24xParquetFileFormat.scala | 11 +++- .../delta/GpuDeltaParquetFileFormat.scala | 11 +++- .../com/nvidia/spark/rapids/GpuExec.scala | 4 ++ .../sql/rapids/GpuFileSourceScanExec.scala | 5 +- 5 files changed, 57 insertions(+), 27 deletions(-) diff --git a/delta-lake/common/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormatUtils.scala b/delta-lake/common/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormatUtils.scala index 101a82da830..1ade53b21b9 100644 --- a/delta-lake/common/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormatUtils.scala +++ b/delta-lake/common/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormatUtils.scala @@ -17,8 +17,8 @@ package com.nvidia.spark.rapids.delta import ai.rapids.cudf.{ColumnVector => CudfColumnVector, Scalar, Table} +import com.nvidia.spark.rapids.{GpuColumnVector, GpuMetric} import com.nvidia.spark.rapids.Arm.{closeOnExcept, withResource} -import com.nvidia.spark.rapids.GpuColumnVector import org.roaringbitmap.longlong.{PeekableLongIterator, Roaring64Bitmap} import org.apache.spark.sql.types.{BooleanType, LongType, StringType, StructField, StructType} @@ -53,7 +53,9 @@ object GpuDeltaParquetFileFormatUtils { schema: StructType, delVector: Option[Roaring64Bitmap], input: Iterator[ColumnarBatch], - maxBatchSize: Int): Iterator[ColumnarBatch] = { + maxBatchSize: Int, + delVectorScatterTimeMetric: GpuMetric + ): Iterator[ColumnarBatch] = { val metadataRowIndexCol = schema.fieldNames.indexOf(METADATA_ROW_IDX_COL) val delRowIdx = schema.fieldNames.indexOf(METADATA_ROW_DEL_COL) if (metadataRowIndexCol == -1 && delRowIdx == -1) { @@ -74,20 +76,31 @@ object GpuDeltaParquetFileFormatUtils { Some(delRowIdx) } val newBatch = addMetadataColumns(rowIdxCol, delRowIdx2, delVector,maxBatchSize, - rowIndex, batch) + rowIndex, batch, delVectorScatterTimeMetric) rowIndex += batch.numRows() newBatch } } } + private def createFalseTable(numRows: Int): Table = { + withResource(Scalar.fromBool(false)) { s => + withResource(CudfColumnVector.fromScalar(s, numRows)) { c => + new Table(c) + } + } + } + + private def addMetadataColumns( rowIdxPos: Option[Int], delRowIdx: Option[Int], delVec: Option[Roaring64Bitmap], maxBatchSize: Int, rowIdxStart: Long, - batch: ColumnarBatch): ColumnarBatch = { + batch: ColumnarBatch, + delVectorScatterTimeMetric: GpuMetric, + ): ColumnarBatch = { val rowIdxCol = rowIdxPos.map { _ => withResource(Scalar.fromLong(rowIdxStart)) { start => GpuColumnVector.from(CudfColumnVector.sequence(start, batch.numRows()), @@ -98,30 +111,26 @@ object GpuDeltaParquetFileFormatUtils { closeOnExcept(rowIdxCol) { rowIdxCol => val delVecCol = delVec.map { delVec => - withResource(Scalar.fromBool(false)) { s => - withResource(CudfColumnVector.fromScalar(s, batch.numRows())) { c => - var table = new Table(c) - val posIter = new RoaringBitmapIterator( - delVec.getLongIteratorFrom(rowIdxStart), - rowIdxStart, - rowIdxStart + batch.numRows(), - ).grouped(Math.min(maxBatchSize, batch.numRows())) - - for (posChunk <- posIter) { - withResource(CudfColumnVector.fromLongs(posChunk: _*)) { poses => - withResource(Scalar.fromBool(true)) { s => - table = withResource(table) { _ => + delVectorScatterTimeMetric.ns { + val table = new RoaringBitmapIterator( + delVec.getLongIteratorFrom(rowIdxStart), + rowIdxStart, + rowIdxStart + batch.numRows()) + .grouped(Math.min(maxBatchSize, batch.numRows())) + .foldLeft(createFalseTable(batch.numRows())){ (table, posChunk) => + withResource(table) { _ => + withResource(CudfColumnVector.fromLongs(posChunk: _*)) { poses => + withResource(Scalar.fromBool(true)) { s => Table.scatter(Array(s), poses, table) } } } } - withResource(table) { _ => - GpuColumnVector.from(table.getColumn(0).incRefCount(), - METADATA_ROW_DEL_FIELD.dataType) - } - } + withResource(table) { _ => + GpuColumnVector.from(table.getColumn(0).incRefCount(), + METADATA_ROW_DEL_FIELD.dataType) + } } } diff --git a/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala b/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala index ef579d78e6f..77891864537 100644 --- a/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala +++ b/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala @@ -88,15 +88,22 @@ case class GpuDelta24xParquetFileFormat( val maxDelVecScatterBatchSize = RapidsConf .DELTA_LOW_SHUFFLE_MERGE_SCATTER_DEL_VECTOR_BATCH_SIZE .get(sparkSession.sessionState.conf) + val delVecScatterTimeMetric = metrics(GpuMetric.DELETION_VECTOR_SCATTER_TIME) + val delVecSizeMetric = metrics(GpuMetric.DELETION_VECTOR_SIZE) + (file: PartitionedFile) => { val input = dataReader(file) val dv = delVecs.flatMap(_.value.get(new URI(file.filePath.toString()))) - .map(dv => RoaringBitmapWrapper.deserializeFromBytes(dv.descriptor.inlineData).inner) + .map { dv => + delVecSizeMetric += dv.descriptor.inlineData.length + RoaringBitmapWrapper.deserializeFromBytes(dv.descriptor.inlineData).inner + } addMetadataColumnToIterator(prepareSchema(requiredSchema), dv, input.asInstanceOf[Iterator[ColumnarBatch]], - maxDelVecScatterBatchSize) + maxDelVecScatterBatchSize, + delVecScatterTimeMetric) .asInstanceOf[Iterator[InternalRow]] } } diff --git a/delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormat.scala b/delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormat.scala index 604ed826397..e109b81f1e5 100644 --- a/delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormat.scala +++ b/delta-lake/delta-spark341db/src/main/scala/com/nvidia/spark/rapids/delta/GpuDeltaParquetFileFormat.scala @@ -85,14 +85,21 @@ case class GpuDeltaParquetFileFormat( .DELTA_LOW_SHUFFLE_MERGE_SCATTER_DEL_VECTOR_BATCH_SIZE .get(sparkSession.sessionState.conf) + val delVecScatterTimeMetric = metrics(GpuMetric.DELETION_VECTOR_SCATTER_TIME) + val delVecSizeMetric = metrics(GpuMetric.DELETION_VECTOR_SIZE) + (file: PartitionedFile) => { val input = dataReader(file) val dv = delVecs.flatMap(_.value.get(new URI(file.filePath.toString()))) - .map(dv => RoaringBitmapWrapper.deserializeFromBytes(dv.descriptor.inlineData).inner) + .map { dv => + delVecSizeMetric += dv.descriptor.inlineData.length + RoaringBitmapWrapper.deserializeFromBytes(dv.descriptor.inlineData).inner + } addMetadataColumnToIterator(prepareSchema(requiredSchema), dv, input.asInstanceOf[Iterator[ColumnarBatch]], - maxDelVecScatterBatchSize + maxDelVecScatterBatchSize, + delVecScatterTimeMetric ).asInstanceOf[Iterator[InternalRow]] } } diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExec.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExec.scala index e93ac40b5bd..9b60cd1efca 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExec.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExec.scala @@ -83,6 +83,8 @@ object GpuMetric extends Logging { val FILECACHE_DATA_RANGE_MISSES_SIZE = "filecacheDataRangeMissesSize" val FILECACHE_FOOTER_READ_TIME = "filecacheFooterReadTime" val FILECACHE_DATA_RANGE_READ_TIME = "filecacheDataRangeReadTime" + val DELETION_VECTOR_SCATTER_TIME = "deletionVectorScatterTime" + val DELETION_VECTOR_SIZE = "deletionVectorSize" // Metric Descriptions. val DESCRIPTION_BUFFER_TIME = "buffer time" @@ -117,6 +119,8 @@ object GpuMetric extends Logging { val DESCRIPTION_FILECACHE_DATA_RANGE_MISSES_SIZE = "cached data misses size" val DESCRIPTION_FILECACHE_FOOTER_READ_TIME = "cached footer read time" val DESCRIPTION_FILECACHE_DATA_RANGE_READ_TIME = "cached data read time" + val DESCRIPTION_DELETION_VECTOR_SCATTER_TIME = "deletion vector scatter time" + val DESCRIPTION_DELETION_VECTOR_SIZE = "deletion vector size" def unwrap(input: GpuMetric): SQLMetric = input match { case w :WrappedGpuMetric => w.sqlMetric diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileSourceScanExec.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileSourceScanExec.scala index a326006ab83..7195580ba69 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileSourceScanExec.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileSourceScanExec.scala @@ -424,7 +424,10 @@ case class GpuFileSourceScanExec( "filesSize" -> createSizeMetric(ESSENTIAL_LEVEL, "size of files read"), GPU_DECODE_TIME -> createNanoTimingMetric(MODERATE_LEVEL, DESCRIPTION_GPU_DECODE_TIME), BUFFER_TIME -> createNanoTimingMetric(MODERATE_LEVEL, DESCRIPTION_BUFFER_TIME), - FILTER_TIME -> createNanoTimingMetric(DEBUG_LEVEL, DESCRIPTION_FILTER_TIME) + FILTER_TIME -> createNanoTimingMetric(DEBUG_LEVEL, DESCRIPTION_FILTER_TIME), + DELETION_VECTOR_SCATTER_TIME -> createNanoTimingMetric(MODERATE_LEVEL, + DESCRIPTION_DELETION_VECTOR_SCATTER_TIME), + DELETION_VECTOR_SIZE -> createSizeMetric(MODERATE_LEVEL, DESCRIPTION_DELETION_VECTOR_SIZE) ) ++ fileCacheMetrics ++ { relation.fileFormat match { case _: GpuReadParquetFileFormat | _: GpuOrcFileFormat => From befb3a59aabf6d436b4905e211c544aa365bb166 Mon Sep 17 00:00:00 2001 From: Liangcai Li Date: Wed, 10 Jul 2024 21:45:06 +0800 Subject: [PATCH 73/79] fix the bucketed write error for non-utc cases (#11164) Signed-off-by: Firestarman --- integration_tests/src/main/python/datasourcev2_write_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_tests/src/main/python/datasourcev2_write_test.py b/integration_tests/src/main/python/datasourcev2_write_test.py index 27c2be481d3..1f4bc133d2a 100644 --- a/integration_tests/src/main/python/datasourcev2_write_test.py +++ b/integration_tests/src/main/python/datasourcev2_write_test.py @@ -15,7 +15,7 @@ import pytest from asserts import assert_gpu_fallback_collect, assert_equal_with_local_sort -from data_gen import gen_df, decimal_gens +from data_gen import gen_df, decimal_gens, non_utc_allow from marks import * from spark_session import is_hive_available, is_spark_330_or_later, with_cpu_session, with_gpu_session from hive_parquet_write_test import _hive_bucket_gens, _hive_array_gens, _hive_struct_gens @@ -30,6 +30,7 @@ @pytest.mark.skipif(not (is_hive_available() and is_spark_330_or_later()), reason="Must have Hive on Spark 3.3+") @pytest.mark.parametrize('file_format', ['parquet', 'orc']) +@allow_non_gpu(*non_utc_allow) def test_write_hive_bucketed_table(spark_tmp_table_factory, file_format): num_rows = 2048 From aede72f3f8083f0861a3dc180f704b51dda6c509 Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Wed, 10 Jul 2024 13:52:29 -0500 Subject: [PATCH 74/79] Coalesce batches after a logical coalesce operation (#11126) Signed-off-by: Robert (Bobby) Evans --- .../scala/com/nvidia/spark/rapids/basicPhysicalOperators.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/basicPhysicalOperators.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/basicPhysicalOperators.scala index 7a38ed808da..b1d81e91a8c 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/basicPhysicalOperators.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/basicPhysicalOperators.scala @@ -1281,6 +1281,8 @@ case class GpuCoalesceExec(numPartitions: Int, child: SparkPlan) rdd.coalesce(numPartitions, shuffle = false) } } + + override val coalesceAfter: Boolean = true } object GpuCoalesceExec { From e9d097f9b2465d1bdc0d8694661de9c1e729fa8a Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Thu, 11 Jul 2024 13:17:35 -0500 Subject: [PATCH 75/79] Fix some GpuBroadcastToRowExec by not dropping columns [databricks] (#11165) * Fix some GpuBraodcastToRowExec by not dropping columns Signed-off-by: Robert (Bobby) Evans * Review comments --------- Signed-off-by: Robert (Bobby) Evans --- integration_tests/src/main/python/aqe_test.py | 75 +++++++++++++++++++ .../spark/rapids/GpuTransitionOverrides.scala | 10 +-- .../execution/GpuBroadcastToRowExec.scala | 33 ++------ 3 files changed, 84 insertions(+), 34 deletions(-) diff --git a/integration_tests/src/main/python/aqe_test.py b/integration_tests/src/main/python/aqe_test.py index e16a4eafcab..3e10f6e9148 100755 --- a/integration_tests/src/main/python/aqe_test.py +++ b/integration_tests/src/main/python/aqe_test.py @@ -391,3 +391,78 @@ def run_test(spark): """) assert_gpu_and_cpu_are_equal_collect(run_test, conf=_adaptive_conf) + +# Verify that DPP and AQE can coexist in even some odd cases involving 2 tables with multiple columns +@ignore_order(local=True) +@allow_non_gpu(*aqe_join_with_dpp_fallback) +def test_aqe_join_with_dpp_multi_columns(spark_tmp_path): + conf = copy_and_update(_adaptive_conf, { + "spark.rapids.sql.explain": "ALL", + "spark.rapids.sql.debug.logTransformations": "true"}) + + data_path = spark_tmp_path + '/PARQUET_DATA' + def write_data(spark): + spark.range(100).selectExpr( + "concat('t_name_', id % 9) as t_name", + "concat('t_id_', id % 9) as t_id", + "concat('v_id_', id % 3) as v_id", + "concat('site_', id % 2) as site_id" + ).repartition(200).write.parquet(data_path + "/tests") + spark.range(2000).selectExpr( + "concat('t_id_', id % 9) as t_spec", + "concat('v_id_', id % 3) as v_spec", + "CAST(id as STRING) as extra_1", + "concat('LONG_SITE_NAME_', id % 2) as site_id", + "if (id % 2 == 0, '1990-01-01', '1990-01-02') as day" + ).write.partitionBy("site_id", "day").parquet(data_path + "/infoB") + spark.range(2000).selectExpr( + "CAST(id as STRING) as extra_1", + "concat('v_id_', id % 3) as v_id", + "CAST(id + 10 as STRING) as extra_3", + "if (id % 3 == 0, 'site_0', 'site_3') as site_id", + "if (id % 2 == 0, '1990-01-01', '1990-01-02') as day", + "concat('t_id_', id % 9) as t_id" + ).write.partitionBy("site_id", "day", "t_id").parquet(data_path + "/infoA") + + with_cpu_session(write_data) + + def run_test(spark): + spark.read.parquet(data_path + "/tests/").createOrReplaceTempView("tests") + spark.read.parquet(data_path + "/infoA/").createOrReplaceTempView("infoA") + spark.read.parquet(data_path + "/infoB/").createOrReplaceTempView("infoB") + return spark.sql(""" + WITH tmp AS + ( SELECT + extra_1, + v_id, + site_id, + day, + t_id + FROM infoA WHERE + day >= '1980-01-01' AND + extra_3 != 50 + UNION ALL + SELECT + extra_1, + v_spec as v_id, + CASE + WHEN site_id = 'LONG_SITE_NAME_0' then 'site_0' + WHEN site_id = 'LONG_SITE_NAME_1' then 'site_1' + ELSE site_id + END AS site_id, + day, + t_spec as t_id + FROM infoB + ) + SELECT + a.t_id, + b.t_name, + a.extra_1, + day + FROM tmp a + JOIN tests b ON a.t_id = b.t_id AND a.v_id = b.v_id AND a.site_id = b.site_id + and day = '1990-01-01' + and a.site_id IN ('site_0', 'site_1'); + """) + + assert_gpu_and_cpu_are_equal_collect(run_test, conf=conf) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuTransitionOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuTransitionOverrides.scala index 00192ed775c..465d9b11f0f 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuTransitionOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuTransitionOverrides.scala @@ -27,7 +27,6 @@ import com.nvidia.spark.rapids.shims.{GpuBatchScanExec, SparkShimImpl} import org.apache.spark.SparkContext import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, AttributeReference, Expression, SortOrder} -import org.apache.spark.sql.catalyst.plans.physical.IdentityBroadcastMode import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.adaptive._ @@ -35,7 +34,7 @@ import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec import org.apache.spark.sql.execution.command.{DataWritingCommandExec, ExecutedCommandExec} import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2ScanExecBase, DropTableExec, ShowTablesExec} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, BroadcastExchangeLike, Exchange, ReusedExchangeExec, ShuffleExchangeLike} -import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec, HashedRelationBroadcastMode} +import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec} import org.apache.spark.sql.rapids.{GpuDataSourceScanExec, GpuFileSourceScanExec, GpuShuffleEnv, GpuTaskMetrics} import org.apache.spark.sql.rapids.execution.{ExchangeMappingCache, GpuBroadcastExchangeExec, GpuBroadcastExchangeExecBase, GpuBroadcastToRowExec, GpuCustomShuffleReaderExec, GpuHashJoin, GpuShuffleExchangeExecBase} import org.apache.spark.sql.types.StructType @@ -213,13 +212,8 @@ class GpuTransitionOverrides extends Rule[SparkPlan] { // we can't directly re-use a GPU broadcast exchange to feed a CPU broadcast // join but Spark will sometimes try and do this val keys = output.map { a => a.asInstanceOf[Expression] } - val (index, keyExprs) = b.mode match { - case HashedRelationBroadcastMode(keys, _) => (None, Some(keys)) - case IdentityBroadcastMode => (Some(0), None) - case m => throw new UnsupportedOperationException(s"Unknown broadcast mode $m") - } SparkShimImpl.newBroadcastQueryStageExec(e, - GpuBroadcastToRowExec(keys, b.mode, e.plan)(index, keyExprs)) + GpuBroadcastToRowExec(keys, b.mode, e.plan)) case b: GpuBroadcastExchangeExec => // This should never happen as AQE with a BroadcastQueryStageExec should // only show up on a reused exchange, but just in case we try to do the right diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastToRowExec.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastToRowExec.scala index d794ab76e45..05fd351f3ed 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastToRowExec.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastToRowExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,7 +36,6 @@ import org.apache.spark.sql.catalyst.plans.QueryPlan import org.apache.spark.sql.catalyst.plans.logical.Statistics import org.apache.spark.sql.catalyst.plans.physical.BroadcastMode import org.apache.spark.sql.execution.{SparkPlan, SQLExecution} -import org.apache.spark.sql.execution.joins.HashedRelationBroadcastMode import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.ThreadUtils @@ -46,11 +45,9 @@ import org.apache.spark.util.ThreadUtils case class GpuBroadcastToRowExec( buildKeys: Seq[Expression], broadcastMode: BroadcastMode, - child: SparkPlan)(index: Option[Int], modeKeys: Option[Seq[Expression]]) + child: SparkPlan) extends ShimBroadcastExchangeLike with ShimUnaryExecNode with GpuExec with Logging { - override def otherCopyArgs: Seq[AnyRef] = index :: modeKeys :: Nil - @transient private val timeout: Long = conf.broadcastTimeout @@ -79,37 +76,21 @@ case class GpuBroadcastToRowExec( override def doCanonicalize(): SparkPlan = { val keys = buildKeys.map(k => QueryPlan.normalizeExpressions(k, child.output)) - GpuBroadcastToRowExec(keys, broadcastMode, child.canonicalized)(index, modeKeys) + GpuBroadcastToRowExec(keys, broadcastMode, child.canonicalized) } - private def projectSerializedBatch( serBatch: SerializeConcatHostBuffersDeserializeBatch): Any = { val beforeCollect = System.nanoTime() - // Creates projection to extract target fields from Row, as what Spark does. - // Note that unlike Spark, the GPU broadcast data has not applied the key expressions from - // the HashedRelation, so that is applied here if necessary to ensure the proper values - // are being extracted. The CPU already has the key projections applied in the broadcast - // data and thus does not have similar logic here. - val broadcastModeProject = modeKeys.map { keyExprs => - UnsafeProjection.create(keyExprs) - } - // Deserializes the batch on the host. Then, transforms it to rows and performs row-wise // projection. We should NOT run any device operation on the driver node. val result = withResource(serBatch.hostBatch) { hostBatch => + val projection = UnsafeProjection.create((0 until hostBatch.numCols()).map { idx => + BoundReference(idx, hostBatch.column(idx).dataType, nullable = true) + }.toSeq) hostBatch.rowIterator().asScala.map { row => - val broadcastRow = broadcastModeProject.map(_(row)).getOrElse(row) - broadcastMode match { - case HashedRelationBroadcastMode(_, _) => - broadcastRow.copy() - case _ => - val idx = index.get - val rowProject = UnsafeProjection.create( - BoundReference(idx, buildKeys(idx).dataType, buildKeys(idx).nullable)) - rowProject(broadcastRow).copy().asInstanceOf[InternalRow] - } + projection(row).copy().asInstanceOf[InternalRow] }.toArray // force evaluation so we don't close hostBatch too soon } From 451463f836f39b4b0de8e92bea182cfa5f38c376 Mon Sep 17 00:00:00 2001 From: Chong Gao Date: Fri, 12 Jul 2024 09:05:24 +0800 Subject: [PATCH 76/79] Case when performance improvement: reduce the `copy_if_else` [databricks] (#10951) * case when improvement: avoid copy_if_else Signed-off-by: Chong Gao Signed-off-by: Chong Gao Co-authored-by: Chong Gao --- .../src/main/python/conditionals_test.py | 87 ++++++++++++++- .../spark/rapids/GpuColumnVectorUtils.scala | 102 +++++++++++++++++- .../nvidia/spark/rapids/GpuOverrides.scala | 2 +- .../com/nvidia/spark/rapids/RapidsConf.scala | 10 ++ .../spark/rapids/conditionalExpressions.scala | 72 ++++++++++--- 5 files changed, 253 insertions(+), 20 deletions(-) diff --git a/integration_tests/src/main/python/conditionals_test.py b/integration_tests/src/main/python/conditionals_test.py index f2dd21b0bf3..b95ed53f398 100644 --- a/integration_tests/src/main/python/conditionals_test.py +++ b/integration_tests/src/main/python/conditionals_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ import pytest -from asserts import assert_gpu_and_cpu_are_equal_collect +from asserts import assert_gpu_and_cpu_are_equal_collect, assert_gpu_and_cpu_are_equal_sql from data_gen import * from spark_session import is_before_spark_320, is_jvm_charset_utf8 from pyspark.sql.types import * @@ -296,3 +296,86 @@ def test_conditional_with_side_effects_unary_minus(data_gen, ansi_enabled): 'CASE WHEN a > -32768 THEN -a ELSE null END'), conf = {'spark.sql.ansi.enabled': ansi_enabled}) +_case_when_scalars = [ + ['True', 'False', 'null', 'True', 'False'], + ['CAST(1 AS TINYINT)', 'CAST(2 AS TINYINT)', 'CAST(3 AS TINYINT)', 'CAST(4 AS TINYINT)', 'CAST(5 AS TINYINT)'], + ['CAST(1 AS SMALLINT)', 'CAST(2 AS SMALLINT)', 'CAST(3 AS SMALLINT)', 'CAST(4 AS SMALLINT)', 'CAST(5 AS SMALLINT)'], + ['1', '2', '3', '4', '5'], + ['CAST(1 AS BIGINT)', 'CAST(2 AS BIGINT)', 'CAST(3 AS BIGINT)', 'CAST(4 AS BIGINT)', 'CAST(5 AS BIGINT)'], + ['CAST(1.1 AS FLOAT)', 'CAST(2.2 AS FLOAT)', 'CAST(3.3 AS FLOAT)', 'CAST(4.4 AS FLOAT)', 'CAST(5.5 AS FLOAT)'], + ['CAST(1.1 AS DOUBLE)', 'CAST(2.2 AS DOUBLE)', 'CAST(3.3 AS DOUBLE)', 'CAST(4.4 AS DOUBLE)', 'CAST(5.5 AS DOUBLE)'], + ["'str_value1'", "'str_value2'", "'str_value3'", "'str_value4'", "'str_else'"], + ['null', 'CAST(2.2 AS DECIMAL(7,3))', 'CAST(3.3 AS DECIMAL(7,3))', 'CAST(4.4 AS DECIMAL(7,3))', 'CAST(5.5 AS DECIMAL(7,3))'], # null and decimal(7) + ['null', 'CAST(2.2 AS DECIMAL(12,2))', 'CAST(3.3 AS DECIMAL(7,3))', 'CAST(4.4 AS DECIMAL(7,3))', 'CAST(5.5 AS DECIMAL(7,3))'], # decimal(7) and decimal(12) + ['CAST(1.1 AS DECIMAL(12,2))', 'CAST(2.2 AS DECIMAL(12,2))', 'CAST(3.3 AS DECIMAL(20,2))', 'CAST(4.4 AS DECIMAL(12,2))', 'CAST(5.5 AS DECIMAL(12,2))'], # decimal(12) and decimal(20) + ['CAST(1.1 AS DECIMAL(20,2))', 'CAST(2.2 AS DECIMAL(20,2))', 'CAST(3.3 AS DECIMAL(20,2))', 'CAST(4.4 AS DECIMAL(20,2))', 'CAST(5.5 AS DECIMAL(20,2))'], # decimal(20) +] +@pytest.mark.parametrize('case_when_scalars', _case_when_scalars, ids=idfn) +def test_case_when_all_then_values_are_scalars(case_when_scalars): + data_gen = [ + ("a", boolean_gen), + ("b", boolean_gen), + ("c", boolean_gen), + ("d", boolean_gen), + ("e", boolean_gen) + ] + sql = """ + select case + when a then {} + when b then {} + when c then {} + when d then {} + else {} + end + from tab + """ + assert_gpu_and_cpu_are_equal_sql( + lambda spark : gen_df(spark, data_gen), + "tab", + sql.format(case_when_scalars[0], case_when_scalars[1], case_when_scalars[2], case_when_scalars[3], case_when_scalars[4]), + conf = {'spark.rapids.sql.case_when.fuse': 'true'}) + +# test corner cases: +# - when exprs has nulls +# - else expr is null +def test_case_when_all_then_values_are_scalars_with_nulls(): + bool_rows = [(True, False, False, None), + (False, True, True, None), # the second true will enable `when b then null` branch + (False, False, None, None), + (None, None, True, False), + (False, False, False, False), + (None, None, None, None)] + sql = """ + select case + when a then 'aaa' + when b then null + when c then 'ccc' + when d then 'ddd' + else {} + end + from tab + """ + sql_without_else = """ + select case + when a then cast(1.1 as decimal(7,2)) + when b then null + when c then cast(3.3 as decimal(7,2)) + when d then cast(4.4 as decimal(7,2)) + end + from tab + """ + assert_gpu_and_cpu_are_equal_sql( + lambda spark: spark.createDataFrame(bool_rows, "a boolean, b boolean, c boolean, d boolean"), + "tab", + sql.format("'unknown'"), + conf = {'spark.rapids.sql.case_when.fuse': 'true'}) + assert_gpu_and_cpu_are_equal_sql( + lambda spark: spark.createDataFrame(bool_rows, "a boolean, b boolean, c boolean, d boolean"), + "tab", + sql.format("null"), # set else as null + conf = {'spark.rapids.sql.case_when.fuse': 'true'}) + assert_gpu_and_cpu_are_equal_sql( + lambda spark: spark.createDataFrame(bool_rows, "a boolean, b boolean, c boolean, d boolean"), + "tab", + sql_without_else, + conf = {'spark.rapids.sql.case_when.fuse': 'true'}) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuColumnVectorUtils.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuColumnVectorUtils.scala index 9fcbba0f4ff..983153edd60 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuColumnVectorUtils.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuColumnVectorUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,14 @@ package com.nvidia.spark.rapids +import ai.rapids.cudf.{ColumnVector => CudfCV, HostColumnVector, Table} +import com.nvidia.spark.rapids.Arm.withResource import java.lang.reflect.Method +import java.util.function.Consumer -import ai.rapids.cudf.Table - -import org.apache.spark.sql.types.DataType +import org.apache.spark.sql.types._ import org.apache.spark.sql.vectorized.ColumnVector +import org.apache.spark.unsafe.types.UTF8String object GpuColumnVectorUtils { lazy val extractHostColumnsMethod: Method = ShimLoader.loadGpuColumnVector() @@ -38,4 +40,96 @@ object GpuColumnVectorUtils { columnVectors.asInstanceOf[Array[ColumnVector]] } + def isCaseWhenFusionSupportedType(dataType: DataType): Boolean = { + dataType match { + case BooleanType => true + case ByteType => true + case ShortType => true + case IntegerType => true + case LongType => true + case FloatType => true + case DoubleType => true + case StringType => true + case _: DecimalType => true + case _ => false + } + } + + /** + * Create column vector from scalars + * @param scalars literals + * @return column vector for the specified scalars + */ + def createFromScalarList(scalars: Seq[GpuScalar]): CudfCV = { + scalars.head.dataType match { + case BooleanType => + val booleans = scalars.map(s => s.getValue.asInstanceOf[java.lang.Boolean]) + CudfCV.fromBoxedBooleans(booleans: _*) + case ByteType => + val bytes = scalars.map(s => s.getValue.asInstanceOf[java.lang.Byte]) + CudfCV.fromBoxedBytes(bytes: _*) + case ShortType => + val shorts = scalars.map(s => s.getValue.asInstanceOf[java.lang.Short]) + CudfCV.fromBoxedShorts(shorts: _*) + case IntegerType => + val ints = scalars.map(s => s.getValue.asInstanceOf[java.lang.Integer]) + CudfCV.fromBoxedInts(ints: _*) + case LongType => + val longs = scalars.map(s => s.getValue.asInstanceOf[java.lang.Long]) + CudfCV.fromBoxedLongs(longs: _*) + case FloatType => + val floats = scalars.map(s => s.getValue.asInstanceOf[java.lang.Float]) + CudfCV.fromBoxedFloats(floats: _*) + case DoubleType => + val doubles = scalars.map(s => s.getValue.asInstanceOf[java.lang.Double]) + CudfCV.fromBoxedDoubles(doubles: _*) + case StringType => + val utf8Bytes = scalars.map(s => { + val v = s.getValue + if (v == null) { + null + } else { + v.asInstanceOf[UTF8String].getBytes + } + }) + CudfCV.fromUTF8Strings(utf8Bytes: _*) + case dt: DecimalType => + val decimals = scalars.map(s => { + val v = s.getValue + if (v == null) { + null + } else { + v.asInstanceOf[Decimal].toJavaBigDecimal + } + }) + fromDecimals(dt, decimals: _*) + case _ => + throw new UnsupportedOperationException(s"Creating column vector from a GpuScalar list" + + s" is not supported for type ${scalars.head.dataType}.") + } + } + + /** + * Create decimal column vector according to DecimalType. + * Note: it will create 3 types of column vector according to DecimalType precision + * - Decimal 32 bits + * - Decimal 64 bits + * - Decimal 128 bits + * E.g.: If the max of values are decimal 32 bits, but DecimalType is 128 bits, + * then return a Decimal 128 bits column vector + */ + def fromDecimals(dt: DecimalType, values: java.math.BigDecimal*): CudfCV = { + val hcv = HostColumnVector.build( + DecimalUtil.createCudfDecimal(dt), + values.length, + new Consumer[HostColumnVector.Builder]() { + override def accept(b: HostColumnVector.Builder): Unit = { + b.appendBoxed(values: _*) + } + } + ) + withResource(hcv) { _ => + hcv.copyToDevice() + } + } } diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 3ec8d2c1d08..f7b71d96ce6 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -2028,7 +2028,7 @@ object GpuOverrides extends Logging { } else { None } - GpuCaseWhen(branches, elseValue) + GpuCaseWhen(branches, elseValue, conf.caseWhenFuseEnabled) } }), expr[If]( diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index 0a8ce614d83..8abf7e616c2 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -2346,6 +2346,14 @@ val SHUFFLE_COMPRESSION_LZ4_CHUNK_SIZE = conf("spark.rapids.shuffle.compression. .stringConf .createOptional + val CASE_WHEN_FUSE = + conf("spark.rapids.sql.case_when.fuse") + .doc("If when branches is greater than 2 and all then/else values in case when are string " + + "scalar, fuse mode improves the performance. By default this is enabled.") + .internal() + .booleanConf + .createWithDefault(true) + private def printSectionHeader(category: String): Unit = println(s"\n### $category") @@ -3171,6 +3179,8 @@ class RapidsConf(conf: Map[String, String]) extends Logging { lazy val loreDumpPath: Option[String] = get(LORE_DUMP_PATH) + lazy val caseWhenFuseEnabled: Boolean = get(CASE_WHEN_FUSE) + private val optimizerDefaults = Map( // this is not accurate because CPU projections do have a cost due to appending values // to each row that is produced, but this needs to be a really small number because diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/conditionalExpressions.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/conditionalExpressions.scala index 0a5e0daa9ff..18a168db422 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/conditionalExpressions.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/conditionalExpressions.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ package com.nvidia.spark.rapids import ai.rapids.cudf.{BinaryOp, ColumnVector, DType, NullPolicy, Scalar, ScanAggregation, ScanType, Table, UnaryOp} import com.nvidia.spark.rapids.Arm._ import com.nvidia.spark.rapids.RapidsPluginImplicits._ +import com.nvidia.spark.rapids.jni.CaseWhen import com.nvidia.spark.rapids.shims.ShimExpression import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion} @@ -47,7 +48,7 @@ object GpuExpressionWithSideEffectUtils { /** * Used to shortcircuit predicates and filter conditions. - * + * * @param nullsAsFalse when true, null values are considered false. * @param col the input being evaluated. * @return boolean. When nullsAsFalse is set, it returns True if none of the rows is true; @@ -182,9 +183,9 @@ case class GpuIf( predicateExpr: Expression, trueExpr: Expression, falseExpr: Expression) extends GpuConditionalExpression { - + import GpuExpressionWithSideEffectUtils._ - + @transient override lazy val inputTypesForMerging: Seq[DataType] = { Seq(trueExpr.dataType, falseExpr.dataType) @@ -314,7 +315,9 @@ case class GpuIf( case class GpuCaseWhen( branches: Seq[(Expression, Expression)], - elseValue: Option[Expression] = None) extends GpuConditionalExpression with Serializable { + elseValue: Option[Expression] = None, + caseWhenFuseEnabled: Boolean = true) + extends GpuConditionalExpression with Serializable { import GpuExpressionWithSideEffectUtils._ @@ -334,6 +337,10 @@ case class GpuCaseWhen( branches.exists(_._2.nullable) || elseValue.forall(_.nullable) } + private lazy val useFusion = caseWhenFuseEnabled && branches.size > 2 && + GpuColumnVectorUtils.isCaseWhenFusionSupportedType(inputTypesForMerging.head) && + (branches.map(_._2) ++ elseValue).forall(_.isInstanceOf[GpuLiteral]) + override def checkInputDataTypes(): TypeCheckResult = { if (TypeCoercion.haveSameType(inputTypesForMerging)) { // Make sure all branch conditions are boolean types. @@ -359,15 +366,54 @@ case class GpuCaseWhen( if (branchesWithSideEffects) { columnarEvalWithSideEffects(batch) } else { - // `elseRet` will be closed in `computeIfElse`. - val elseRet = elseValue - .map(_.columnarEvalAny(batch)) - .getOrElse(GpuScalar(null, branches.last._2.dataType)) - val any = branches.foldRight[Any](elseRet) { - case ((predicateExpr, trueExpr), falseRet) => - computeIfElse(batch, predicateExpr, trueExpr, falseRet) + if (useFusion) { + // when branches size > 2; + // return type is supported types: Boolean/Byte/Int/String/Decimal ... + // all the then and else expressions are Scalars. + // Avoid to use multiple `computeIfElse`s which will create multiple temp columns + + // 1. select first true index from bool columns, if no true, index will be out of bound + // e.g.: + // case when bool result column 0: true, false, false + // case when bool result column 1: false, true, false + // result is: [0, 1, 2] + val whenBoolCols = branches.safeMap(_._1.columnarEval(batch).getBase).toArray + val firstTrueIndex: ColumnVector = withResource(whenBoolCols) { _ => + CaseWhen.selectFirstTrueIndex(whenBoolCols) + } + + withResource(firstTrueIndex) { _ => + val thenElseScalars = (branches.map(_._2) ++ elseValue).map(_.columnarEvalAny(batch) + .asInstanceOf[GpuScalar]) + withResource(thenElseScalars) { _ => + // 2. generate a column to store all scalars + withResource(GpuColumnVectorUtils.createFromScalarList(thenElseScalars)) { + scalarCol => + val finalRet = withResource(new Table(scalarCol)) { oneColumnTable => + // 3. execute final select + // default gather OutOfBoundsPolicy is nullify, + // If index is out of bound, return null + withResource(oneColumnTable.gather(firstTrueIndex)) { resultTable => + resultTable.getColumn(0).incRefCount() + } + } + // return final column vector + GpuColumnVector.from(finalRet, dataType) + } + } + } + } else { + // execute from tail to front recursively + // `elseRet` will be closed in `computeIfElse`. + val elseRet = elseValue + .map(_.columnarEvalAny(batch)) + .getOrElse(GpuScalar(null, branches.last._2.dataType)) + val any = branches.foldRight[Any](elseRet) { + case ((predicateExpr, trueExpr), falseRet) => + computeIfElse(batch, predicateExpr, trueExpr, falseRet) + } + GpuExpressionsUtils.resolveColumnVector(any, batch.numRows()) } - GpuExpressionsUtils.resolveColumnVector(any, batch.numRows()) } } From be34c6a97004d46ca2541e214552fecb5a172ddd Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Fri, 12 Jul 2024 19:16:05 +0800 Subject: [PATCH 77/79] Drop spark31x shims [databricks] (#11159) * Remove spark31x json lines and shim files 1, Remove spark31x json lines from the source code 2, Remove the files those only for spark31x shims 3, Move the files for spark31x and spark32x+ shims into sql-plugin/src/main/spark320 folder Signed-off-by: Tim Liu * Drop spark31x shims in the build scripts and pom files Signed-off-by: Tim Liu * Restore the accidentally deleted file: OrcStatisticShim.scala tests/src/test/spark311/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala --> tests/src/test/spark321cdh/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala check if we chan merge this file into? tests/src/test/spark320/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala Signed-off-by: Tim Liu * Update Copyright to 2024 Signed-off-by: Tim Liu * Remove the 31x in ShimLoader.scala according to the review comments Signed-off-by: Tim Liu * Update the file scala2.13/pom.xml Signed-off-by: Tim Liu * Drop 3.1.x shims in docs, source code and build scripts Change the default shim to spark320 from spark311 in the shims in docs, source code and build scripts Signed-off-by: Tim Liu * Updating the docs for the dropping 31x shims Signed-off-by: Tim Liu * Clean up unused and duplicated 'org/roaringbitmap' folder To fix: https://github.com/NVIDIA/spark-rapids/issues/11175 Clean up unused and duplicated 'org/roaringbitmap' in the spark320 shim folder to walk around for the JACOCO error 'different class with same name', after we drop 31x shims and change the default shim to spark320 Signed-off-by: Tim Liu --------- Signed-off-by: Tim Liu --- CONTRIBUTING.md | 21 +- aggregator/pom.xml | 70 +- api_validation/README.md | 2 +- api_validation/auditAllVersions.sh | 4 +- .../spark/rapids/api/ApiValidation.scala | 4 +- build/buildall | 6 +- build/coverage-report | 2 +- build/make-scala-version-build-files.sh | 6 +- build/shimplify.py | 8 +- .../sql/tests/datagen/DataGenExprBase.scala | 27 - dist/README.md | 10 +- dist/build/package-parallel-worlds.py | 8 +- dist/maven-antrun/build-parallel-worlds.xml | 2 +- dist/pom.xml | 3 +- dist/scripts/binary-dedupe.sh | 8 +- ...txt => unshimmed-common-from-spark320.txt} | 0 .../advanced_configs.md | 8 +- docs/dev/README.md | 4 +- docs/dev/shimplify.md | 14 +- docs/dev/shims.md | 4 +- docs/supported_ops.md | 2904 ++++++++++++++--- jenkins/hadoop-def.sh | 4 +- jenkins/spark-nightly-build.sh | 4 +- jenkins/spark-premerge-build.sh | 9 +- jenkins/spark-tests.sh | 2 +- jenkins/version-def.sh | 4 +- pom.xml | 74 +- scala2.13/aggregator/pom.xml | 70 +- scala2.13/dist/pom.xml | 3 +- scala2.13/pom.xml | 72 +- .../com/nvidia/spark/rapids/ShimLoader.scala | 8 +- .../com/nvidia/spark/rapids/RapidsConf.scala | 6 +- ...mmedExecutionPlanCaptureCallbackImpl.scala | 4 +- .../shims/ShimSupportsRuntimeFiltering.java | 35 - .../spark/rapids/shims/XxHash64Shims.scala | 30 - .../shims/AvoidAdaptiveTransitionToRow.scala | 89 - .../rapids/shims/BatchScanExecMeta.scala | 39 - .../spark/rapids/shims/GpuBatchScanExec.scala | 59 - .../spark/rapids/shims/GpuDataSourceRDD.scala | 68 - .../spark/rapids/shims/GpuOrcDataReader.scala | 100 - .../spark/rapids/shims/GpuParquetCrypto.scala | 28 - .../nvidia/spark/rapids/shims/HashUtils.scala | 33 - .../shims/OffsetWindowFunctionMeta.scala | 87 - .../spark/rapids/shims/OrcCastingShims.scala | 35 - .../nvidia/spark/rapids/shims/OrcShims.scala | 43 - .../shims/OrcShims311until320Base.scala | 133 - .../rapids/shims/ParquetSchemaClipShims.scala | 150 - .../spark/rapids/shims/PlanShimsImpl.scala | 43 - .../rapids/shims/RapidsOrcScanMeta.scala | 51 - .../rapids/shims/RapidsParquetScanMeta.scala | 52 - .../rapids/shims/ShimAQEShuffleReadExec.scala | 76 - .../rapids/shims/ShimBaseSubqueryExec.scala | 29 - .../shims/ShimBroadcastExchangeLike.scala | 46 - .../rapids/shims/ShimPredicateHelper.scala | 44 - .../rapids/shims/ShuffleOriginUtil.scala | 31 - .../spark/rapids/shims/Spark31XShims.scala | 441 --- .../spark/rapids/shims/SparkShims.scala | 44 - .../nvidia/spark/rapids/shims/TreeNode.scala | 47 - .../spark/rapids/shims/TypeSigUtil.scala | 77 - .../spark/rapids/shims/YearParseUtil.scala | 29 - .../spark/rapids/shims/gpuWindows.scala | 65 - .../spark311/SparkShimServiceProvider.scala | 36 - .../spark311/RapidsShuffleManager.scala | 30 - .../shims/GpuShuffleBlockResolver.scala | 31 - .../rapids/shims/ShuffledBatchRDDUtil.scala | 120 - .../shims/storage/ShimDiskBlockManager.scala | 29 - .../shims/ShimVectorizedColumnReader.scala | 69 - .../rapids/execution/ShimTrampolineUtil.scala | 41 - .../python/shims/GpuArrowPythonOutput.scala | 90 - .../spark/sql/rapids/shims/AvroUtils.scala | 36 - .../rapids/shims/GpuJsonToStructsShim.scala | 82 - .../sql/rapids/shims/RapidsErrorUtils.scala | 84 - .../rapids/shims/RapidsQueryErrorUtils.scala | 125 - .../shims/RapidsShuffleThreadedReader.scala | 66 - .../shims/RapidsShuffleThreadedWriter.scala | 55 - .../rapids/shims/datetimeExpressions.scala | 41 - .../types/shims/PartitionValueCastShims.scala | 36 - .../RapidsShuffleBlockFetcherIterator.scala | 1050 ------ .../spark/rapids/shims/SparkShims.scala | 44 - .../spark312/SparkShimServiceProvider.scala | 36 - .../spark312/RapidsShuffleManager.scala | 30 - .../spark/rapids/shims/SparkShims.scala | 49 - .../spark313/SparkShimServiceProvider.scala | 36 - .../spark313/RapidsShuffleManager.scala | 29 - .../nvidia/spark/rapids/shims/AQEUtils.scala | 3 - .../rapids/shims/AggregationTagging.scala | 3 - .../spark/rapids/shims/AnsiCastShim.scala | 3 - .../nvidia/spark/rapids/shims/AnsiUtil.scala | 3 - .../spark/rapids/shims/BloomFilterShims.scala | 3 - .../rapids/shims/BucketSpecForHiveShim.scala | 3 - .../rapids/shims/BucketingUtilsShim.scala | 3 - .../spark/rapids/shims/CastCheckShims.scala | 3 - .../rapids/shims/CastingConfigShim.scala | 3 - .../rapids/shims/CharVarcharUtilsShims.scala | 3 - .../shims/ColumnDefaultValuesShims.scala | 3 - ...aSourceTableAsSelectCommandMetaShims.scala | 3 - .../spark/rapids/shims/CudfUnsafeRow.scala | 3 - .../rapids/shims/CudfUnsafeRowBase.scala | 3 - .../rapids/shims/DateTimeUtilsShims.scala | 3 - .../shims/DecimalArithmeticOverrides.scala | 3 - .../rapids/shims/DecimalMultiply128.scala | 3 - .../spark/rapids/shims/DeltaLakeUtils.scala | 3 - .../spark/rapids/shims/DistributionUtil.scala | 5 +- .../rapids/shims/FileIndexOptionsShims.scala | 3 - .../spark/rapids/shims/GetMapValueMeta.scala | 3 - .../spark/rapids/shims/GetSequenceSize.scala | 3 - .../spark/rapids/shims/GlobalLimitShims.scala | 3 - .../shims/GpuAggregateInPandasExecMeta.scala | 3 - .../rapids/shims/GpuBroadcastJoinMeta.scala | 3 - .../spark/rapids/shims/GpuCastShims.scala | 3 - .../shims/GpuFileFormatDataWriterShim.scala | 3 - .../rapids/shims/GpuHashPartitioning.scala | 5 +- .../spark/rapids/shims/GpuIntervalUtils.scala | 3 - ...dCreateHiveTableAsSelectCommandShims.scala | 3 - .../rapids/shims/GpuRangePartitioning.scala | 5 +- .../spark/rapids/shims/GpuTypeShims.scala | 3 - .../rapids/shims/GpuWindowInPandasExec.scala | 3 - .../spark/rapids/shims/InSubqueryShims.scala | 3 - .../shims/LegacyBehaviorPolicyShim.scala | 3 - .../rapids/shims/NullOutputStreamShim.scala | 3 - .../rapids/shims/OrcProtoWriterShim.scala | 3 - .../spark/rapids/shims/OrcReadingShims.scala | 3 - .../rapids/shims/ParquetFieldIdShims.scala | 5 +- .../shims/ParquetLegacyNanoAsLongShims.scala | 3 - .../rapids/shims/ParquetStringPredShims.scala | 3 - .../shims/ParquetTimestampNTZShims.scala | 3 - .../shims/PartitionedFileUtilsShim.scala | 5 +- .../spark/rapids/shims/PythonUDFShim.scala | 3 - .../spark/rapids/shims/RaiseErrorShim.scala | 3 - .../shims/RapidsFileSourceMetaUtils.scala | 3 - .../ShimFilePartitionReaderFactory.scala | 3 - .../spark/rapids/shims/ShimLeafExecNode.scala | 3 - .../rapids/shims/Spark31Xuntil33XShims.scala | 3 - .../spark/rapids/shims/TypeUtilsShims.scala | 3 - .../rapids/shims/extractValueShims.scala | 3 - .../shuffle/RapidsShuffleIterator.scala | 3 - .../rapids/shims/GpuShuffleExchangeExec.scala | 3 - .../spark/sql/catalyst/csv/GpuCsvUtils.scala | 5 +- .../sql/catalyst/json/GpuJsonUtils.scala | 5 +- .../parquet/ShimCurrentBatchIterator.scala | 3 - .../rapids/shims/FilePartitionShims.scala | 3 - .../hive/rapids/shims/CommandUtilsShim.scala | 3 - .../hive/rapids/shims/FileSinkDescShim.scala | 3 - .../GpuCreateHiveTableAsSelectCommand.scala | 3 - .../rapids/shims/GpuInsertIntoHiveTable.scala | 3 - .../shims/GpuRowBasedHiveGenericUDFShim.scala | 3 - .../rapids/shims/HiveInspectorsShim.scala | 3 - .../rapids/shims/HiveProviderCmdShims.scala | 3 - .../spark/sql/rapids/GpuDataSource.scala | 3 - .../sql/rapids/GpuFileFormatWriter.scala | 3 - .../sql/rapids/RapidsCachingReader.scala | 3 - .../rapids/aggregate/aggregateFunctions.scala | 3 - .../apache/spark/sql/rapids/arithmetic.scala | 3 - .../execution/GpuBroadcastHashJoinExec.scala | 3 - .../GpuBroadcastNestedLoopJoinExec.scala | 3 - .../sql/rapids/execution/GpuShuffleMeta.scala | 3 - .../execution/GpuSubqueryBroadcastMeta.scala | 3 - .../python/shims/GpuArrowPythonRunner.scala | 3 - .../python/shims/GpuBasePythonRunner.scala | 3 - .../shims/GpuCoGroupedArrowPythonRunner.scala | 3 - .../shims/GpuGroupedPythonRunnerFactory.scala | 3 - .../python/shims/WritePythonUDFUtils.scala | 3 - .../sql/rapids/shims/ArrowUtilsShim.scala | 3 - .../sql/rapids/shims/DataTypeUtilsShim.scala | 3 - .../spark/sql/rapids/shims/GpuAscii.scala | 3 - ...eDataSourceTableAsSelectCommandShims.scala | 3 - .../rapids/shims/GpuMapInPandasExecMeta.scala | 3 - .../shims/RapidsHadoopWriterUtils.scala | 3 - .../sql/rapids/shims/SchemaUtilsShims.scala | 3 - .../shims/SparkUpgradeExceptionShims.scala | 3 - .../apache/spark/sql/rapids/shims/misc.scala | 5 +- .../shims/spark311/SparkShimsSuite.scala | 34 - .../shims/spark312/SparkShimsSuite.scala | 34 - .../shims/spark313/SparkShimsSuite.scala | 34 - tests/README.md | 4 +- .../rapids/timezone/TimeZonePerfSuite.scala | 2 +- .../shuffle/RapidsShuffleTestHelper.scala | 3 - .../spark/rapids/shims/OrcStatisticShim.scala | 3 - tools/generated_files/operatorsScore.csv | 6 +- tools/generated_files/supportedDataSource.csv | 26 +- tools/generated_files/supportedExecs.csv | 104 +- tools/generated_files/supportedExprs.csv | 1484 ++++----- 182 files changed, 3364 insertions(+), 6271 deletions(-) delete mode 100644 datagen/src/main/spark311/scala/org/apache/spark/sql/tests/datagen/DataGenExprBase.scala rename dist/{unshimmed-common-from-spark311.txt => unshimmed-common-from-spark320.txt} (100%) delete mode 100644 sql-plugin/src/main/spark311/java/com/nvidia/spark/rapids/shims/ShimSupportsRuntimeFiltering.java delete mode 100644 sql-plugin/src/main/spark311/java/com/nvidia/spark/rapids/shims/XxHash64Shims.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AvoidAdaptiveTransitionToRow.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuDataSourceRDD.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuParquetCrypto.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/HashUtils.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcCastingShims.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcShims.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcShims311until320Base.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/PlanShimsImpl.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RapidsOrcScanMeta.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RapidsParquetScanMeta.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimAQEShuffleReadExec.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimBaseSubqueryExec.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimBroadcastExchangeLike.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimPredicateHelper.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShuffleOriginUtil.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/Spark31XShims.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/SparkShims.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/TreeNode.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/YearParseUtil.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala delete mode 100644 sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/spark311/RapidsShuffleManager.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleBlockResolver.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/ShuffledBatchRDDUtil.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/storage/ShimDiskBlockManager.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/ShimVectorizedColumnReader.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/ShimTrampolineUtil.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/AvroUtils.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuJsonToStructsShim.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsQueryErrorUtils.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedReader.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedWriter.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/datetimeExpressions.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/types/shims/PartitionValueCastShims.scala delete mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/storage/RapidsShuffleBlockFetcherIterator.scala delete mode 100644 sql-plugin/src/main/spark312/scala/com/nvidia/spark/rapids/shims/SparkShims.scala delete mode 100644 sql-plugin/src/main/spark312/scala/com/nvidia/spark/rapids/shims/spark312/SparkShimServiceProvider.scala delete mode 100644 sql-plugin/src/main/spark312/scala/com/nvidia/spark/rapids/spark312/RapidsShuffleManager.scala delete mode 100644 sql-plugin/src/main/spark313/scala/com/nvidia/spark/rapids/shims/SparkShims.scala delete mode 100644 sql-plugin/src/main/spark313/scala/com/nvidia/spark/rapids/shims/spark313/SparkShimServiceProvider.scala delete mode 100644 sql-plugin/src/main/spark313/scala/com/nvidia/spark/rapids/spark313/RapidsShuffleManager.scala rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/AQEUtils.scala (97%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala (95%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/AnsiCastShim.scala (97%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/AnsiUtil.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/BloomFilterShims.scala (95%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/BucketingUtilsShim.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/CastCheckShims.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/CastingConfigShim.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/CharVarcharUtilsShims.scala (95%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/ColumnDefaultValuesShims.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRowBase.scala (99%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/DateTimeUtilsShims.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/DecimalArithmeticOverrides.scala (99%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/DecimalMultiply128.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/DeltaLakeUtils.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/DistributionUtil.scala (93%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/FileIndexOptionsShims.scala (95%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/GetMapValueMeta.scala (97%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/GetSequenceSize.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/GlobalLimitShims.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/GpuAggregateInPandasExecMeta.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/GpuBroadcastJoinMeta.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/GpuCastShims.scala (95%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/GpuFileFormatDataWriterShim.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/GpuHashPartitioning.scala (94%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtils.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala (99%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/GpuRangePartitioning.scala (97%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/GpuTypeShims.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/GpuWindowInPandasExec.scala (97%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/InSubqueryShims.scala (95%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala (95%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/OrcProtoWriterShim.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/OrcReadingShims.scala (97%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/ParquetFieldIdShims.scala (93%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/ParquetLegacyNanoAsLongShims.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/ParquetStringPredShims.scala (95%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/ParquetTimestampNTZShims.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala (93%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/PythonUDFShim.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala (97%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/RapidsFileSourceMetaUtils.scala (95%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/ShimFilePartitionReaderFactory.scala (95%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/ShimLeafExecNode.scala (95%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/Spark31Xuntil33XShims.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/TypeUtilsShims.scala (95%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shims/extractValueShims.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala (99%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala (91%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/catalyst/json/GpuJsonUtils.scala (95%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/execution/datasources/parquet/ShimCurrentBatchIterator.scala (99%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/execution/rapids/shims/FilePartitionShims.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/hive/rapids/shims/FileSinkDescShim.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala (99%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala (99%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/hive/rapids/shims/GpuRowBasedHiveGenericUDFShim.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/hive/rapids/shims/HiveInspectorsShim.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/hive/rapids/shims/HiveProviderCmdShims.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/GpuDataSource.scala (99%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala (99%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/RapidsCachingReader.scala (99%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/aggregate/aggregateFunctions.scala (97%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/arithmetic.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastNestedLoopJoinExec.scala (99%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/execution/GpuShuffleMeta.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuBasePythonRunner.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala (98%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala (97%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/shims/ArrowUtilsShim.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/shims/DataTypeUtilsShim.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala (97%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala (99%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/shims/GpuMapInPandasExecMeta.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/shims/RapidsHadoopWriterUtils.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/shims/SchemaUtilsShims.scala (97%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala (96%) rename sql-plugin/src/main/{spark311 => spark320}/scala/org/apache/spark/sql/rapids/shims/misc.scala (95%) delete mode 100644 sql-plugin/src/test/spark311/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimsSuite.scala delete mode 100644 sql-plugin/src/test/spark312/scala/com/nvidia/spark/rapids/shims/spark312/SparkShimsSuite.scala delete mode 100644 sql-plugin/src/test/spark313/scala/com/nvidia/spark/rapids/shims/spark313/SparkShimsSuite.scala rename tests/src/test/{spark311 => spark320}/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleTestHelper.scala (99%) rename tests/src/test/{spark311 => spark321cdh}/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala (97%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 295006be49c..c52516023f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,11 +50,11 @@ mvn verify After a successful build, the RAPIDS Accelerator jar will be in the `dist/target/` directory. This will build the plugin for a single version of Spark. By default, this is Apache Spark -3.1.1. To build against other versions of Spark you use the `-Dbuildver=XXX` command line option -to Maven. For instance to build Spark 3.1.1 you would use: +3.2.0. To build against other versions of Spark you use the `-Dbuildver=XXX` command line option +to Maven. For instance to build Spark 3.2.0 you would use: ```shell script -mvn -Dbuildver=311 verify +mvn -Dbuildver=320 verify ``` You can find all available build versions in the top level pom.xml file. If you are building for Databricks then you should use the `jenkins/databricks/build.sh` script and modify it for @@ -110,7 +110,7 @@ If you want to create a jar with multiple versions we have the following options 3. Build for all Apache Spark versions, CDH and Databricks with no SNAPSHOT versions of Spark, only released. Use `-PnoSnaphsotsWithDatabricks`. 4. Build for all Apache Spark versions, CDH and Databricks including SNAPSHOT versions of Spark we have supported for. Use `-PsnapshotsWithDatabricks` 5. Build for an arbitrary combination of comma-separated build versions using `-Dincluded_buildvers=`. - E.g., `-Dincluded_buildvers=312,330` + E.g., `-Dincluded_buildvers=320,330` You must first build each of the versions of Spark and then build one final time using the profile for the option you want. @@ -118,9 +118,6 @@ You can also install some manually and build a combined jar. For instance to bui ```shell script mvn clean -mvn -Dbuildver=311 install -Drat.skip=true -DskipTests -mvn -Dbuildver=312 install -Drat.skip=true -DskipTests -mvn -Dbuildver=313 install -Drat.skip=true -DskipTests mvn -Dbuildver=320 install -Drat.skip=true -DskipTests mvn -Dbuildver=321 install -Drat.skip=true -DskipTests mvn -Dbuildver=321cdh install -Drat.skip=true -DskipTests @@ -150,9 +147,9 @@ There is a build script `build/buildall` that automates the local build process. By default, it builds everything that is needed to create a distribution jar for all released (noSnapshots) Spark versions except for Databricks. Other profiles that you can pass using `--profile=` include - `snapshots` that includes all released (noSnapshots) and snapshots Spark versions except for Databricks -- `minimumFeatureVersionMix` that currently includes 321cdh, 312, 320, 330 is recommended for catching incompatibilities already in the local development cycle +- `minimumFeatureVersionMix` that currently includes 321cdh, 320, 330 is recommended for catching incompatibilities already in the local development cycle -For initial quick iterations we can use `--profile=` to build a single-shim version. e.g., `--profile=311` for Spark 3.1.1. +For initial quick iterations we can use `--profile=` to build a single-shim version. e.g., `--profile=320` for Spark 3.2.0. The option `--module=` allows to limit the number of build steps. When iterating, we often don't have the need for the entire build. We may be interested in building everything necessary just to run integration tests (`--module=integration_tests`), or we may want to just rebuild the distribution jar (`--module=dist`) @@ -201,7 +198,7 @@ NOTE: Build process does not require an ARM machine, so if you want to build the on X86 machine, please also add `-DskipTests` in commands. ```bash -mvn clean verify -Dbuildver=311 -Parm64 +mvn clean verify -Dbuildver=320 -Parm64 ``` ### Iterative development during local testing @@ -377,7 +374,7 @@ the symlink `.bloop` to point to the corresponding directory `.bloop-spark3XY` Example usage: ```Bash -./build/buildall --generate-bloop --profile=311,330 +./build/buildall --generate-bloop --profile=320,330 rm -vf .bloop ln -s .bloop-spark330 .bloop ``` @@ -414,7 +411,7 @@ Install [Scala Metals extension](https://scalameta.org/metals/docs/editors/vscod either locally or into a Remote-SSH extension destination depending on your target environment. When your project folder is open in VS Code, it may prompt you to import Maven project. IMPORTANT: always decline with "Don't ask again", otherwise it will overwrite the Bloop projects -generated with the default `311` profile. If you need to use a different profile, always rerun the +generated with the default `320` profile. If you need to use a different profile, always rerun the command above manually. When regenerating projects it's recommended to proceed to Metals "Build commands" View, and click: 1. "Restart build server" diff --git a/aggregator/pom.xml b/aggregator/pom.xml index 8cf881419c9..4f0ea3f6c16 100644 --- a/aggregator/pom.xml +++ b/aggregator/pom.xml @@ -252,79 +252,11 @@ - release311 + release320 true - - buildver - 311 - - - - - com.nvidia - rapids-4-spark-delta-stub_${scala.binary.version} - ${project.version} - ${spark.version.classifier} - - - - - release312 - - - buildver - 312 - - - - - com.nvidia - rapids-4-spark-delta-stub_${scala.binary.version} - ${project.version} - ${spark.version.classifier} - - - - - release313 - - - buildver - 313 - - - - - com.nvidia - rapids-4-spark-delta-stub_${scala.binary.version} - ${project.version} - ${spark.version.classifier} - - - - - release314 - - - buildver - 314 - - - - - com.nvidia - rapids-4-spark-delta-stub_${scala.binary.version} - ${project.version} - ${spark.version.classifier} - - - - - release320 - buildver 320 diff --git a/api_validation/README.md b/api_validation/README.md index 5f34354656d..482b3e76a58 100644 --- a/api_validation/README.md +++ b/api_validation/README.md @@ -21,7 +21,7 @@ cd api_validation sh auditAllVersions.sh // To run script on particular version we can use profile -mvn scala:run -P spark311 +mvn scala:run -P spark320 ``` # Output diff --git a/api_validation/auditAllVersions.sh b/api_validation/auditAllVersions.sh index 5deddacec65..27aeedcd4ba 100644 --- a/api_validation/auditAllVersions.sh +++ b/api_validation/auditAllVersions.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,4 +14,4 @@ # limitations under the License. set -ex -mvn scala:run -P spark311 +mvn scala:run -P spark320 diff --git a/api_validation/src/main/scala/com/nvidia/spark/rapids/api/ApiValidation.scala b/api_validation/src/main/scala/com/nvidia/spark/rapids/api/ApiValidation.scala index 58d273d2148..5821c6e18ac 100644 --- a/api_validation/src/main/scala/com/nvidia/spark/rapids/api/ApiValidation.scala +++ b/api_validation/src/main/scala/com/nvidia/spark/rapids/api/ApiValidation.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -69,7 +69,7 @@ object ApiValidation extends Logging { val gpuKeys = gpuExecs.keys var printNewline = false - val sparkToShimMap = Map("3.1.1" -> "spark311") + val sparkToShimMap = Map("3.2.0" -> "spark320") val sparkVersion = ShimLoader.getShimVersion.toString val shimVersion = sparkToShimMap(sparkVersion) diff --git a/build/buildall b/build/buildall index b3c473be141..3c3c8a045ce 100755 --- a/build/buildall +++ b/build/buildall @@ -274,8 +274,8 @@ export -f build_single_shim # Install all the versions for DIST_PROFILE # First build the aggregator module for all SPARK_SHIM_VERSIONS in parallel skipping expensive plugins that -# - either deferred to 311 because the check is identical in all shim profiles such as scalastyle -# - or deferred to 311 because we currently don't require it per shim such as scaladoc generation +# - either deferred to 320 because the check is identical in all shim profiles such as scalastyle +# - or deferred to 320 because we currently don't require it per shim such as scaladoc generation # - or there is a dedicated step to run against a particular shim jar such as unit tests, in # the near future we will run unit tests against a combined multi-shim jar to catch classloading # regressions even before pytest-based integration_tests @@ -296,7 +296,7 @@ time ( fi # This used to resume from dist. However, without including aggregator in the build # the build does not properly initialize spark.version property via buildver profiles - # in the root pom, and we get a missing spark311 dependency even for --profile=312,321 + # in the root pom, and we get a missing spark320 dependency even for --profile=320,321 # where the build does not require it. Moving it to aggregator resolves this issue with # a negligible increase of the build time by ~2 seconds. joinShimBuildFrom="aggregator" diff --git a/build/coverage-report b/build/coverage-report index 75c86e55258..ddde86b6aea 100755 --- a/build/coverage-report +++ b/build/coverage-report @@ -23,7 +23,7 @@ TMP_CLASS=${TEMP_CLASS_LOC:-"./target/jacoco_classes/"} HTML_LOC=${HTML_LOCATION:="./target/jacoco-report/"} XML_LOC=${XML_LOCATION:="${HTML_LOC}"} DIST_JAR=${RAPIDS_DIST_JAR:-$(ls ./dist/target/rapids-4-spark_2.12-*cuda*.jar | grep -v test | head -1 | xargs readlink -f)} -SPK_VER=${JACOCO_SPARK_VER:-"311"} +SPK_VER=${JACOCO_SPARK_VER:-"320"} UDF_JAR=${RAPIDS_UDF_JAR:-$(ls ./udf-compiler/target/spark${SPK_VER}/rapids-4-spark-udf_2.12-*-SNAPSHOT-spark${SPK_VER}.jar | grep -v test | head -1 | xargs readlink -f)} SOURCE_DIRS=${SOURCE_DIRS:-"./sql-plugin/src/main/scala/:./sql-plugin/src/main/java/:./shuffle-plugin/src/main/scala/:./udf-compiler/src/main/scala/"} diff --git a/build/make-scala-version-build-files.sh b/build/make-scala-version-build-files.sh index c1ca50b0551..ad3482ee979 100755 --- a/build/make-scala-version-build-files.sh +++ b/build/make-scala-version-build-files.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ set -e VALID_VERSIONS=( 2.13 ) declare -A DEFAULT_SPARK -DEFAULT_SPARK[2.12]="spark311" +DEFAULT_SPARK[2.12]="spark320" DEFAULT_SPARK[2.13]="spark330" usage() { @@ -94,4 +94,4 @@ sed_i '//,/[0-9]*\.[0-9]*/,/[0-9]*\.[0-9]*\.[0-9]*[0-9]*\.[0-9]*\.[0-9]*'$SCALA_VERSION' -You can find all shim files for a particular shim, e.g. 312, easily by executing: -git grep '{"spark": "312"}' '*.java' '*.scala' +You can find all shim files for a particular shim, e.g. 320, easily by executing: +git grep '{"spark": "320"}' '*.java' '*.scala' """ import errno diff --git a/datagen/src/main/spark311/scala/org/apache/spark/sql/tests/datagen/DataGenExprBase.scala b/datagen/src/main/spark311/scala/org/apache/spark/sql/tests/datagen/DataGenExprBase.scala deleted file mode 100644 index d50008f7fb7..00000000000 --- a/datagen/src/main/spark311/scala/org/apache/spark/sql/tests/datagen/DataGenExprBase.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.tests.datagen - -import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, UnaryExpression} -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback - -trait DataGenExprBase extends UnaryExpression with ExpectsInputTypes with CodegenFallback diff --git a/dist/README.md b/dist/README.md index 56e5b9be297..2a0955da6dd 100644 --- a/dist/README.md +++ b/dist/README.md @@ -17,21 +17,21 @@ Files are: `com.nvidia.spark.rapids.SparkShimServiceProvider.sparkNonSnapshot`, The new uber jar is structured like: -1. Base common classes are user visible classes. For these we use Spark 3.1.1 versions because they are assumed to be +1. Base common classes are user visible classes. For these we use Spark 3.2.0 versions because they are assumed to be bitwise-identical to the other shims, this assumption is subject to the future automatic validation. 2. META-INF/services. This is a file that has to list all the shim versions supported by this jar. The files talked about above for each profile are put into place here for uber jars. Although we currently do not use [ServiceLoader API](https://docs.oracle.com/javase/8/docs/api/java/util/ServiceLoader.html) we use the same service provider discovery mechanism -3. META-INF base files are from 3.1.1 - maven, LICENSE, NOTICE, etc +3. META-INF base files are from 3.2.0 - maven, LICENSE, NOTICE, etc 4. Spark specific directory (aka Parallel World in the jargon of [ParallelWorldClassloader](https://github.com/openjdk/jdk/blob/jdk8-b120/jaxws/src/share/jaxws_classes/com/sun/istack/internal/tools/ParallelWorldClassLoader.java)) -for each version of Spark supported in the jar, i.e., spark311/, spark312/, spark320/, etc. +for each version of Spark supported in the jar, i.e., spark320/, spark330/, spark341/, etc. If you have to change the contents of the uber jar the following files control what goes into the base jar as classes that are not shaded. -1. `unshimmed-common-from-spark311.txt` - This has classes and files that should go into the base jar with their normal +1. `unshimmed-common-from-spark320.txt` - This has classes and files that should go into the base jar with their normal package name (not shaded). This includes user visible classes (i.e., com/nvidia/spark/SQLPlugin), python files, -and other files that aren't version specific. Uses Spark 3.1.1 built jar for these base classes as explained above. +and other files that aren't version specific. Uses Spark 3.2.0 built jar for these base classes as explained above. 2. `unshimmed-from-each-spark3xx.txt` - This is applied to all the individual Spark specific version jars to pull any files that need to go into the base of the jar and not into the Spark specific directory. diff --git a/dist/build/package-parallel-worlds.py b/dist/build/package-parallel-worlds.py index 568ad5ca55f..ef64a4cd6bd 100644 --- a/dist/build/package-parallel-worlds.py +++ b/dist/build/package-parallel-worlds.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -73,8 +73,8 @@ def shell_exec(shell_cmd): shell_exec(mvn_cmd) dist_dir = os.sep.join([source_basedir, 'dist']) - with open(os.sep.join([dist_dir, 'unshimmed-common-from-spark311.txt']), 'r') as f: - from_spark311 = f.read().splitlines() + with open(os.sep.join([dist_dir, 'unshimmed-common-from-spark320.txt']), 'r') as f: + from_spark320 = f.read().splitlines() with open(os.sep.join([dist_dir, 'unshimmed-from-each-spark3xx.txt']), 'r') as f: from_each = f.read().splitlines() with zipfile.ZipFile(os.sep.join([deps_dir, art_jar]), 'r') as zip_handle: @@ -88,7 +88,7 @@ def shell_exec(shell_cmd): # TODO deprecate namelist = zip_handle.namelist() matching_members = [] - glob_list = from_spark311 + from_each if bv == buildver_list[0] else from_each + glob_list = from_spark320 + from_each if bv == buildver_list[0] else from_each for pat in glob_list: new_matches = fnmatch.filter(namelist, pat) matching_members += new_matches diff --git a/dist/maven-antrun/build-parallel-worlds.xml b/dist/maven-antrun/build-parallel-worlds.xml index 07838616340..bc4d7c9991c 100644 --- a/dist/maven-antrun/build-parallel-worlds.xml +++ b/dist/maven-antrun/build-parallel-worlds.xml @@ -132,7 +132,7 @@ + includesfile="${spark.rapids.source.basedir}/${rapids.module}/unshimmed-common-from-spark320.txt"/> diff --git a/dist/pom.xml b/dist/pom.xml index ecc2018baea..814b2bf63db 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -145,7 +145,6 @@ minimumFeatureVersionMix - 312, 320, 321cdh, 330, @@ -389,7 +388,7 @@ self.log("... OK") - + diff --git a/dist/scripts/binary-dedupe.sh b/dist/scripts/binary-dedupe.sh index 970840f59e3..3a9df845568 100755 --- a/dist/scripts/binary-dedupe.sh +++ b/dist/scripts/binary-dedupe.sh @@ -131,14 +131,14 @@ mv "$SPARK_SHARED_DIR" parallel-world/ # identical regardless of the Spark-version-specific jar. # # At this point the duplicate classes have not been removed from version-specific jar -# locations such as parallel-world/spark312. +# locations such as parallel-world/spark321. # For each unshimmed class file look for all of its copies inside /spark[34]* and # and count the number of distinct checksums. There are two representative cases # 1) The class is contributed to the unshimmed location via the unshimmed-from-each-spark34 list. These are classes # carrying the shim classifier in their package name such as -# com.nvidia.spark.rapids.spark312.RapidsShuffleManager. They are unique by construction, -# and will have zero copies in any non-spark312 shims. Although such classes are currently excluded from -# being copied to the /spark312 Parallel World we keep the algorithm below general without assuming this. +# com.nvidia.spark.rapids.spark321.RapidsShuffleManager. They are unique by construction, +# and will have zero copies in any non-spark321 shims. Although such classes are currently excluded from +# being copied to the /spark321 Parallel World we keep the algorithm below general without assuming this. # # 2) The class is contributed to the unshimmed location via unshimmed-common. These are classes that # that have the same package and class name across all parallel worlds. diff --git a/dist/unshimmed-common-from-spark311.txt b/dist/unshimmed-common-from-spark320.txt similarity index 100% rename from dist/unshimmed-common-from-spark311.txt rename to dist/unshimmed-common-from-spark320.txt diff --git a/docs/additional-functionality/advanced_configs.md b/docs/additional-functionality/advanced_configs.md index 2f76ffd8a68..12d2fda4ec6 100644 --- a/docs/additional-functionality/advanced_configs.md +++ b/docs/additional-functionality/advanced_configs.md @@ -329,7 +329,7 @@ Name | SQL Function(s) | Description | Default Value | Notes spark.rapids.sql.expression.PromotePrecision| |PromotePrecision before arithmetic operations between DecimalType data|true|None| spark.rapids.sql.expression.PythonUDF| |UDF run in an external python process. Does not actually run on the GPU, but the transfer of data to/from it can be accelerated|true|None| spark.rapids.sql.expression.Quarter|`quarter`|Returns the quarter of the year for date, in the range 1 to 4|true|None| -spark.rapids.sql.expression.RLike|`rlike`|Regular expression version of Like|true|None| +spark.rapids.sql.expression.RLike|`regexp_like`, `regexp`, `rlike`|Regular expression version of Like|true|None| spark.rapids.sql.expression.RaiseError|`raise_error`|Throw an exception|true|None| spark.rapids.sql.expression.Rand|`rand`, `random`|Generate a random column with i.i.d. uniformly distributed values in [0, 1)|true|None| spark.rapids.sql.expression.Rank|`rank`|Window function that returns the rank value within the aggregation window|true|None| @@ -438,14 +438,18 @@ Name | Description | Default Value | Notes spark.rapids.sql.exec.SubqueryBroadcastExec|Plan to collect and transform the broadcast key values|true|None| spark.rapids.sql.exec.TakeOrderedAndProjectExec|Take the first limit elements as defined by the sortOrder, and do projection if needed|true|None| spark.rapids.sql.exec.UnionExec|The backend for the union operator|true|None| -spark.rapids.sql.exec.CustomShuffleReaderExec|A wrapper of shuffle query stage|true|None| +spark.rapids.sql.exec.AQEShuffleReadExec|A wrapper of shuffle query stage|true|None| spark.rapids.sql.exec.HashAggregateExec|The backend for hash based aggregations|true|None| spark.rapids.sql.exec.ObjectHashAggregateExec|The backend for hash based aggregations supporting TypedImperativeAggregate functions|true|None| spark.rapids.sql.exec.SortAggregateExec|The backend for sort based aggregations|true|None| spark.rapids.sql.exec.InMemoryTableScanExec|Implementation of InMemoryTableScanExec to use GPU accelerated caching|true|None| spark.rapids.sql.exec.DataWritingCommandExec|Writing data|true|None| spark.rapids.sql.exec.ExecutedCommandExec|Eagerly executed commands|true|None| +spark.rapids.sql.exec.AppendDataExecV1|Append data into a datasource V2 table using the V1 write interface|true|None| +spark.rapids.sql.exec.AtomicCreateTableAsSelectExec|Create table as select for datasource V2 tables that support staging table creation|true|None| +spark.rapids.sql.exec.AtomicReplaceTableAsSelectExec|Replace table as select for datasource V2 tables that support staging table creation|true|None| spark.rapids.sql.exec.BatchScanExec|The backend for most file input|true|None| +spark.rapids.sql.exec.OverwriteByExpressionExecV1|Overwrite into a datasource V2 table using the V1 write interface|true|None| spark.rapids.sql.exec.BroadcastExchangeExec|The backend for broadcast exchange of data|true|None| spark.rapids.sql.exec.ShuffleExchangeExec|The backend for most data being exchanged between processes|true|None| spark.rapids.sql.exec.BroadcastHashJoinExec|Implementation of join using broadcast data|true|None| diff --git a/docs/dev/README.md b/docs/dev/README.md index 9307780494d..ecbd0252ae7 100644 --- a/docs/dev/README.md +++ b/docs/dev/README.md @@ -271,9 +271,9 @@ is not currently [able to support](https://github.com/jacoco/jacoco/issues/965) setup. So if you want to generate a coverage report you need to do it manually. Coverage is collected by default so first run the tests, and then generate the report, this should be run from the root project directory. It will print out the URL of the report at the end. Besides, -coverage report only covers test with Spark 311 by default as [jacoco](https://www.jacoco.org/jacoco/trunk/doc/) +coverage report only covers test with Spark 320 by default as [jacoco](https://www.jacoco.org/jacoco/trunk/doc/) can't support combined jars. If you're testing with different Spark version, please change it -via environment variable `JACOCO_SPARK_VER` before generate coverage report, e.g, `export JACOCO_SPARK_VER=311`. +via environment variable `JACOCO_SPARK_VER` before generate coverage report, e.g, `export JACOCO_SPARK_VER=320`. ```bash mvn clean verify diff --git a/docs/dev/shimplify.md b/docs/dev/shimplify.md index 94d8a97bec7..a8f075016ae 100644 --- a/docs/dev/shimplify.md +++ b/docs/dev/shimplify.md @@ -28,7 +28,7 @@ time of this writing) have a new set of special sibling directories `src/(main|test)/spark${buildver}`. Previous `src/(main|test)/${buildver}` and -version-range-with-exceptions directories such as `src/main/311until340-non330db` are deprecated and +version-range-with-exceptions directories such as `src/main/320until340-non330db` are deprecated and are being removed as a result of the conversion to the new structure. `shimplify` changes the way the source code is shared among shims by using an explicit @@ -37,7 +37,7 @@ in a source-code level comment instead of the shared directories. ```scala /*** spark-rapids-shim-json-lines -{"spark": "312"} +{"spark": "320"} {"spark": "323"} spark-rapids-shim-json-lines ***/ ``` @@ -155,7 +155,7 @@ It is not expected to be really necessary but it is possible to convert a subset * Either by adding -Dshimplify.shims=buildver1,buildver2,... to the commands above * Or by specifying a list of directories you would like to delete to have a simpler directory --Dshimplify.dirs=311until340-non330db,320until330-noncdh +-Dshimplify.dirs=320until340-non330db,320until330-noncdh The latter is just a minor twist on the former. Instead of having an explicit list of shims, it first computes the list of all `buildver` values using provided directories. After this *all* the @@ -202,15 +202,15 @@ work on resolving potential compilation failures manually. ## Deleting a Shim -Every Spark build is de-supported eventually. To drop a build say 311 you can run +Every Spark build is de-supported eventually. To drop a build say 320 you can run ```bash mvn generate-sources -Dshimplify=true -Dshimplify.move=true \ - -Dshimplify.remove.shim=311 + -Dshimplify.remove.shim=320 ``` -This command will remove the comment line `{"spark": "311"}` from all source files contributing to -the 311 shim. If a file belongs exclusively to 311 it will be removed. +This command will remove the comment line `{"spark": "320"}` from all source files contributing to +the 320 shim. If a file belongs exclusively to 320 it will be removed. After adding or deleting shims you should sanity-check the diff in the local git repo and run the integration tests above. diff --git a/docs/dev/shims.md b/docs/dev/shims.md index 218adbc99d4..a9317fa33c2 100644 --- a/docs/dev/shims.md +++ b/docs/dev/shims.md @@ -8,7 +8,7 @@ parent: Developer Overview # Shim Development RAPIDS Accelerator For Apache Spark supports multiple feature version lines of -Apache Spark such as 3.1.x, 3.2.x, 3.3.0 and a number of vendor releases that contain +Apache Spark such as 3.2.x, 3.3.x, 3.4.x, 3.5.x and a number of vendor releases that contain a mix of patches from different upstream releases. These artifacts are generally incompatible between each other, at both source code level and even more often at the binary level. The role of the Shim layer is to hide these issues from the @@ -159,7 +159,7 @@ to build against the lowest and highest versions of the supported Spark version range. As of the time of this writing: ```bash -./build/buildall --parallel=4 --profile=311,330 --module=dist +./build/buildall --parallel=4 --profile=320,351 --module=dist ``` However, before submitting the PR execute the full build `--profile=noSnapshots`. diff --git a/docs/supported_ops.md b/docs/supported_ops.md index 018d6640976..4bf7e77c0ac 100644 --- a/docs/supported_ops.md +++ b/docs/supported_ops.md @@ -9,7 +9,7 @@ support all data types. The RAPIDS Accelerator for Apache Spark has further restrictions on what types are supported for processing. This tries to document what operations are supported and what data types each operation supports. Because Apache Spark is under active development too and this document was generated -against version 3.1.1 of Spark. Most of this should still +against version 3.2.0 of Spark. Most of this should still apply to other versions of Spark, but there may be slight changes. # General limitations @@ -128,6 +128,8 @@ Accelerator supports are described below. MAP STRUCT UDT +DAYTIME +YEARMONTH CoalesceExec @@ -148,9 +150,11 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -172,9 +176,11 @@ Accelerator supports are described below. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -196,9 +202,11 @@ Accelerator supports are described below. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -220,9 +228,11 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -244,9 +254,11 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -268,9 +280,11 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -292,9 +306,11 @@ Accelerator supports are described below. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -316,9 +332,11 @@ Accelerator supports are described below. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -340,9 +358,11 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -368,6 +388,8 @@ Accelerator supports are described below. + + SampleExec @@ -388,9 +410,11 @@ Accelerator supports are described below. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -412,9 +436,11 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -440,6 +466,8 @@ Accelerator supports are described below. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S TakeOrderedAndProjectExec @@ -460,9 +488,11 @@ Accelerator supports are described below. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -484,9 +514,11 @@ Accelerator supports are described below. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
unionByName will not optionally impute nulls for missing struct fields when the column is a struct and there are non-overlapping fields;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
unionByName will not optionally impute nulls for missing struct fields when the column is a struct and there are non-overlapping fields;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -512,9 +544,11 @@ Accelerator supports are described below. MAP STRUCT UDT +DAYTIME +YEARMONTH -CustomShuffleReaderExec +AQEShuffleReadExec A wrapper of shuffle query stage None Input/Output @@ -532,9 +566,11 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -556,9 +592,11 @@ Accelerator supports are described below. S PS
not allowed for grouping expressions NS -PS
not allowed for grouping expressions if containing Struct as child;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
not allowed for grouping expressions;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
not allowed for grouping expressions if containing Array, Map, or Binary as child;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
not allowed for grouping expressions if containing Struct as child;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
not allowed for grouping expressions;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
not allowed for grouping expressions if containing Array, Map, or Binary as child;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -580,9 +618,11 @@ Accelerator supports are described below. S PS
not allowed for grouping expressions and only allowed when aggregate buffers can be converted between CPU and GPU NS -PS
not allowed for grouping expressions;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
not allowed for grouping expressions;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
not allowed for grouping expressions if containing Array, Map, or Binary as child;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
not allowed for grouping expressions;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
not allowed for grouping expressions;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
not allowed for grouping expressions if containing Array, Map, or Binary as child;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -604,9 +644,11 @@ Accelerator supports are described below. S PS
not allowed for grouping expressions and only allowed when aggregate buffers can be converted between CPU and GPU NS -PS
not allowed for grouping expressions;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
not allowed for grouping expressions;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
not allowed for grouping expressions if containing Array, Map, or Binary as child;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
not allowed for grouping expressions;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
not allowed for grouping expressions;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
not allowed for grouping expressions if containing Array, Map, or Binary as child;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -628,9 +670,11 @@ Accelerator supports are described below. NS NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -652,9 +696,11 @@ Accelerator supports are described below. NS S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -680,6 +726,86 @@ Accelerator supports are described below. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S + + +AppendDataExecV1 +Append data into a datasource V2 table using the V1 write interface +None +Input/Output +S +S +S +S +S +S +S +S +PS
UTC is only supported TZ for TIMESTAMP +S +S +NS +S +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS +NS + + +AtomicCreateTableAsSelectExec +Create table as select for datasource V2 tables that support staging table creation +None +Input/Output +S +S +S +S +S +S +S +S +PS
UTC is only supported TZ for TIMESTAMP +S +S +NS +S +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS +NS + + +AtomicReplaceTableAsSelectExec +Replace table as select for datasource V2 tables that support staging table creation +None +Input/Output +S +S +S +S +S +S +S +S +PS
UTC is only supported TZ for TIMESTAMP +S +S +NS +S +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS +NS BatchScanExec @@ -700,9 +826,37 @@ Accelerator supports are described below. NS S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS +NS + + +OverwriteByExpressionExecV1 +Overwrite into a datasource V2 table using the V1 write interface +None +Input/Output +S +S +S +S +S +S +S +S +PS
UTC is only supported TZ for TIMESTAMP +S +S +NS +S +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -724,9 +878,11 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -748,9 +904,11 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
Round-robin partitioning is not supported if spark.sql.execution.sortBeforeRepartition is true;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
Round-robin partitioning is not supported for nested structs if spark.sql.execution.sortBeforeRepartition is true;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
Round-robin partitioning is not supported if spark.sql.execution.sortBeforeRepartition is true;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
Round-robin partitioning is not supported for nested structs if spark.sql.execution.sortBeforeRepartition is true;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -774,7 +932,9 @@ Accelerator supports are described below. NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -795,7 +955,9 @@ Accelerator supports are described below. NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -818,6 +980,8 @@ Accelerator supports are described below. + + Input/Output @@ -835,12 +999,40 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS +Executor +Description +Notes +Param(s) +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT +DAYTIME +YEARMONTH + + BroadcastNestedLoopJoinExec Implementation of join using brute force. Full outer joins and joins where the broadcast side matches the join side (e.g.: LeftOuter with left broadcast) are not supported None @@ -863,6 +1055,8 @@ Accelerator supports are described below. + + Input/Output @@ -880,34 +1074,12 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS - - -Executor -Description -Notes -Param(s) -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT CartesianProductExec @@ -928,9 +1100,11 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -954,7 +1128,9 @@ Accelerator supports are described below. NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -975,7 +1151,9 @@ Accelerator supports are described below. NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -998,6 +1176,8 @@ Accelerator supports are described below. + + Input/Output @@ -1015,9 +1195,11 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -1041,7 +1223,9 @@ Accelerator supports are described below. NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -1062,7 +1246,9 @@ Accelerator supports are described below. NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -1085,6 +1271,8 @@ Accelerator supports are described below. + + Input/Output @@ -1102,9 +1290,11 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -1130,6 +1320,8 @@ Accelerator supports are described below. NS NS NS +NS +NS ArrowEvalPythonExec @@ -1150,9 +1342,11 @@ Accelerator supports are described below. NS NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT NS @@ -1178,6 +1372,8 @@ Accelerator supports are described below. NS NS NS +NS +NS FlatMapGroupsInPandasExec @@ -1202,6 +1398,34 @@ Accelerator supports are described below. NS NS NS +NS +NS + + +Executor +Description +Notes +Param(s) +BOOLEAN +BYTE +SHORT +INT +LONG +FLOAT +DOUBLE +DATE +TIMESTAMP +STRING +DECIMAL +NULL +BINARY +CALENDAR +ARRAY +MAP +STRUCT +UDT +DAYTIME +YEARMONTH MapInPandasExec @@ -1222,9 +1446,11 @@ Accelerator supports are described below. NS NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT NS @@ -1235,45 +1461,23 @@ Accelerator supports are described below. S S S -S -S -S -S -S -PS
UTC is only supported TZ for TIMESTAMP -S -NS -NS -NS -NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT -NS -NS -NS - - -Executor -Description -Notes -Param(s) -BOOLEAN -BYTE -SHORT -INT -LONG -FLOAT -DOUBLE -DATE -TIMESTAMP -STRING -DECIMAL -NULL -BINARY -CALENDAR -ARRAY -MAP -STRUCT -UDT +S +S +S +S +S +PS
UTC is only supported TZ for TIMESTAMP +S +NS +NS +NS +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH +NS +NS +NS +NS +NS WindowExec @@ -1296,7 +1500,9 @@ Accelerator supports are described below. NS NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -1315,9 +1521,11 @@ Accelerator supports are described below. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -1343,6 +1551,8 @@ Accelerator supports are described below. NS NS NS +NS +NS @@ -1406,6 +1616,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Abs @@ -1432,6 +1644,8 @@ are limited. + + result @@ -1453,6 +1667,8 @@ are limited. + + AST @@ -1475,6 +1691,8 @@ are limited. + + result @@ -1496,6 +1714,8 @@ are limited. + + Acos @@ -1522,6 +1742,8 @@ are limited. + + result @@ -1543,6 +1765,8 @@ are limited. + + AST @@ -1565,6 +1789,8 @@ are limited. + + result @@ -1586,6 +1812,8 @@ are limited. + + Acosh @@ -1612,6 +1840,8 @@ are limited. + + result @@ -1633,6 +1863,8 @@ are limited. + + AST @@ -1655,6 +1887,8 @@ are limited. + + result @@ -1676,6 +1910,8 @@ are limited. + + Add @@ -1702,6 +1938,8 @@ are limited. +NS +NS rhs @@ -1723,6 +1961,8 @@ are limited. +NS +NS result @@ -1744,6 +1984,8 @@ are limited. +NS +NS AST @@ -1766,6 +2008,8 @@ are limited. +NS +NS rhs @@ -1787,6 +2031,8 @@ are limited. +NS +NS result @@ -1808,6 +2054,8 @@ are limited. +NS +NS Expression @@ -1834,6 +2082,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Alias @@ -1856,9 +2106,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -1877,9 +2129,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -1903,6 +2157,8 @@ are limited. NS NS NS +NS +NS result @@ -1924,6 +2180,8 @@ are limited. NS NS NS +NS +NS And @@ -1950,6 +2208,8 @@ are limited. + + rhs @@ -1971,6 +2231,8 @@ are limited. + + result @@ -1992,6 +2254,8 @@ are limited. + + AST @@ -2014,6 +2278,8 @@ are limited. + + rhs @@ -2035,6 +2301,8 @@ are limited. + + result @@ -2056,6 +2324,8 @@ are limited. + + ArrayContains @@ -2078,7 +2348,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH + + @@ -2103,6 +2375,8 @@ are limited. NS NS NS +NS +NS result @@ -2124,6 +2398,8 @@ are limited. + + ArrayExcept @@ -2146,7 +2422,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH + + @@ -2167,7 +2445,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH + + @@ -2188,7 +2468,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH + + @@ -2218,6 +2500,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH ArrayExists @@ -2240,7 +2524,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -2265,6 +2551,8 @@ are limited. + + result @@ -2286,6 +2574,8 @@ are limited. + + ArrayFilter @@ -2308,7 +2598,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -2333,6 +2625,8 @@ are limited. + + result @@ -2350,7 +2644,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -2376,7 +2672,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH + + @@ -2397,7 +2695,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH + + @@ -2418,7 +2718,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH + + @@ -2448,6 +2750,8 @@ are limited. + + result @@ -2469,6 +2773,8 @@ are limited. NS NS + + ArrayMin @@ -2495,6 +2801,8 @@ are limited. + + result @@ -2516,6 +2824,8 @@ are limited. NS NS + + ArrayRemove @@ -2538,7 +2848,9 @@ are limited. NS NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS NS NS @@ -2559,9 +2871,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -2580,7 +2894,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -2610,6 +2926,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH ArrayRepeat @@ -2632,9 +2950,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -2657,6 +2977,8 @@ are limited. + + result @@ -2674,7 +2996,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -2700,7 +3024,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -2721,9 +3047,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -2742,7 +3070,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -2768,7 +3098,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH + + @@ -2789,7 +3121,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH + + @@ -2810,7 +3144,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH + + @@ -2836,7 +3172,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH + + @@ -2857,7 +3195,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH + + @@ -2882,6 +3222,8 @@ are limited. + + ArraysZip @@ -2904,7 +3246,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -2925,7 +3269,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -2955,6 +3301,8 @@ are limited. + + result @@ -2976,6 +3324,8 @@ are limited. + + Expression @@ -3002,6 +3352,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Asin @@ -3028,6 +3380,8 @@ are limited. + + result @@ -3049,6 +3403,8 @@ are limited. + + AST @@ -3071,6 +3427,8 @@ are limited. + + result @@ -3092,6 +3450,8 @@ are limited. + + Asinh @@ -3118,6 +3478,8 @@ are limited. + + result @@ -3139,6 +3501,8 @@ are limited. + + AST @@ -3161,6 +3525,8 @@ are limited. + + result @@ -3182,6 +3548,8 @@ are limited. + + AtLeastNNonNulls @@ -3204,9 +3572,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -3229,6 +3599,8 @@ are limited. + + Atan @@ -3255,6 +3627,8 @@ are limited. + + result @@ -3276,6 +3650,8 @@ are limited. + + AST @@ -3298,6 +3674,8 @@ are limited. + + result @@ -3319,6 +3697,8 @@ are limited. + + Atanh @@ -3345,6 +3725,8 @@ are limited. + + result @@ -3366,6 +3748,8 @@ are limited. + + AST @@ -3388,6 +3772,8 @@ are limited. + + result @@ -3409,6 +3795,8 @@ are limited. + + Expression @@ -3435,6 +3823,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH AttributeReference @@ -3457,9 +3847,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -3483,6 +3875,8 @@ are limited. NS NS NS +NS +NS BRound @@ -3509,6 +3903,8 @@ are limited. + + scale @@ -3530,6 +3926,8 @@ are limited. + + result @@ -3551,6 +3949,8 @@ are limited. + + BitLength @@ -3577,6 +3977,8 @@ are limited. + + result @@ -3598,6 +4000,8 @@ are limited. + + BitwiseAnd @@ -3624,6 +4028,8 @@ are limited. + + rhs @@ -3645,6 +4051,8 @@ are limited. + + result @@ -3666,6 +4074,8 @@ are limited. + + AST @@ -3688,6 +4098,8 @@ are limited. + + rhs @@ -3709,6 +4121,8 @@ are limited. + + result @@ -3730,6 +4144,8 @@ are limited. + + BitwiseNot @@ -3756,6 +4172,8 @@ are limited. + + result @@ -3777,6 +4195,8 @@ are limited. + + AST @@ -3799,6 +4219,8 @@ are limited. + + result @@ -3820,6 +4242,8 @@ are limited. + + Expression @@ -3846,6 +4270,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH BitwiseOr @@ -3872,6 +4298,8 @@ are limited. + + rhs @@ -3893,6 +4321,8 @@ are limited. + + result @@ -3914,6 +4344,8 @@ are limited. + + AST @@ -3936,6 +4368,8 @@ are limited. + + rhs @@ -3957,6 +4391,8 @@ are limited. + + result @@ -3978,6 +4414,8 @@ are limited. + + BitwiseXor @@ -4004,6 +4442,8 @@ are limited. + + rhs @@ -4025,6 +4465,8 @@ are limited. + + result @@ -4046,6 +4488,8 @@ are limited. + + AST @@ -4068,6 +4512,8 @@ are limited. + + rhs @@ -4089,6 +4535,8 @@ are limited. + + result @@ -4110,6 +4558,8 @@ are limited. + + BoundReference @@ -4132,9 +4582,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -4158,6 +4610,8 @@ are limited. NS NS NS +NS +NS CaseWhen @@ -4184,6 +4638,8 @@ are limited. + + value @@ -4201,9 +4657,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -4222,9 +4680,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -4252,6 +4712,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Cbrt @@ -4278,6 +4740,8 @@ are limited. + + result @@ -4299,6 +4763,8 @@ are limited. + + AST @@ -4321,6 +4787,8 @@ are limited. + + result @@ -4342,6 +4810,8 @@ are limited. + + Ceil @@ -4368,6 +4838,8 @@ are limited. + + result @@ -4389,6 +4861,8 @@ are limited. + + CheckOverflow @@ -4415,6 +4889,8 @@ are limited. + + result @@ -4436,6 +4912,8 @@ are limited. + + Coalesce @@ -4458,9 +4936,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -4479,9 +4959,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -4505,7 +4987,9 @@ are limited. NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -4526,7 +5010,9 @@ are limited. NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -4556,6 +5042,8 @@ are limited. + + result @@ -4577,6 +5065,8 @@ are limited. + + Contains @@ -4603,6 +5093,8 @@ are limited. + + search @@ -4624,6 +5116,8 @@ are limited. + + result @@ -4645,6 +5139,8 @@ are limited. + + Expression @@ -4671,6 +5167,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Conv @@ -4697,6 +5195,8 @@ are limited. + + from_base @@ -4718,6 +5218,8 @@ are limited. + + to_base @@ -4739,6 +5241,8 @@ are limited. + + result @@ -4760,6 +5264,8 @@ are limited. + + Cos @@ -4786,6 +5292,8 @@ are limited. + + result @@ -4807,6 +5315,8 @@ are limited. + + AST @@ -4829,6 +5339,8 @@ are limited. + + result @@ -4850,6 +5362,8 @@ are limited. + + Cosh @@ -4876,6 +5390,8 @@ are limited. + + result @@ -4897,6 +5413,8 @@ are limited. + + AST @@ -4919,6 +5437,8 @@ are limited. + + result @@ -4940,6 +5460,8 @@ are limited. + + Cot @@ -4966,6 +5488,8 @@ are limited. + + result @@ -4987,6 +5511,8 @@ are limited. + + AST @@ -5009,6 +5535,8 @@ are limited. + + result @@ -5030,6 +5558,8 @@ are limited. + + Expression @@ -5056,6 +5586,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH CreateArray @@ -5078,9 +5610,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, MAP, UDT NS @@ -5099,7 +5633,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, MAP, UDT, DAYTIME, YEARMONTH + + @@ -5129,6 +5665,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP + + value @@ -5150,6 +5688,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP + + CreateNamedStruct @@ -5176,6 +5716,8 @@ are limited. + + value @@ -5193,9 +5735,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -5216,7 +5760,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -5244,6 +5790,8 @@ are limited. + + DateAdd @@ -5270,6 +5818,8 @@ are limited. + + days @@ -5291,6 +5841,8 @@ are limited. + + result @@ -5312,6 +5864,8 @@ are limited. + + DateAddInterval @@ -5338,6 +5892,8 @@ are limited. + + interval @@ -5359,6 +5915,8 @@ are limited. + + result @@ -5380,6 +5938,8 @@ are limited. + + DateDiff @@ -5406,6 +5966,8 @@ are limited. + + rhs @@ -5427,6 +5989,8 @@ are limited. + + result @@ -5448,6 +6012,8 @@ are limited. + + Expression @@ -5474,6 +6040,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH DateFormatClass @@ -5500,6 +6068,8 @@ are limited. + + strfmt @@ -5521,6 +6091,8 @@ are limited. + + result @@ -5542,6 +6114,8 @@ are limited. + + DateSub @@ -5568,6 +6142,8 @@ are limited. + + days @@ -5589,6 +6165,8 @@ are limited. + + result @@ -5610,6 +6188,8 @@ are limited. + + DayOfMonth @@ -5636,6 +6216,8 @@ are limited. + + result @@ -5657,6 +6239,8 @@ are limited. + + DayOfWeek @@ -5683,6 +6267,8 @@ are limited. + + result @@ -5704,6 +6290,8 @@ are limited. + + DayOfYear @@ -5730,6 +6318,8 @@ are limited. + + result @@ -5751,6 +6341,8 @@ are limited. + + DenseRank @@ -5777,6 +6369,8 @@ are limited. NS NS NS +NS +NS result @@ -5798,6 +6392,8 @@ are limited. + + Divide @@ -5824,6 +6420,8 @@ are limited. + + rhs @@ -5845,6 +6443,8 @@ are limited. + + result @@ -5866,6 +6466,8 @@ are limited. + + Expression @@ -5892,6 +6494,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH DynamicPruningExpression @@ -5918,6 +6522,8 @@ are limited. + + result @@ -5939,6 +6545,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S ElementAt @@ -5961,8 +6569,10 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
If it's map, only primitive key types are supported.;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
If it's map, only primitive key types are supported.;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -5986,6 +6596,8 @@ are limited. NS NS NS +NS +NS result @@ -6003,9 +6615,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -6033,6 +6647,8 @@ are limited. + + search @@ -6054,6 +6670,8 @@ are limited. + + result @@ -6075,6 +6693,8 @@ are limited. + + EqualNullSafe @@ -6101,6 +6721,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + rhs @@ -6122,6 +6744,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + result @@ -6143,6 +6767,8 @@ are limited. + + EqualTo @@ -6169,6 +6795,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + rhs @@ -6190,6 +6818,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + result @@ -6211,6 +6841,8 @@ are limited. + + AST @@ -6233,6 +6865,8 @@ are limited. NS NS + + rhs @@ -6254,6 +6888,8 @@ are limited. NS NS + + result @@ -6275,6 +6911,8 @@ are limited. + + Expression @@ -6301,6 +6939,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Exp @@ -6327,6 +6967,8 @@ are limited. + + result @@ -6348,6 +6990,8 @@ are limited. + + AST @@ -6370,6 +7014,8 @@ are limited. + + result @@ -6391,6 +7037,8 @@ are limited. + + Explode @@ -6413,8 +7061,10 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -6434,7 +7084,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -6464,6 +7116,8 @@ are limited. + + result @@ -6485,6 +7139,8 @@ are limited. + + AST @@ -6507,6 +7163,8 @@ are limited. + + result @@ -6528,6 +7186,8 @@ are limited. + + Flatten @@ -6554,6 +7214,8 @@ are limited. + + result @@ -6575,6 +7237,8 @@ are limited. + + Floor @@ -6601,6 +7265,8 @@ are limited. + + result @@ -6622,6 +7288,8 @@ are limited. + + FormatNumber @@ -6648,6 +7316,8 @@ are limited. + + d @@ -6669,6 +7339,8 @@ are limited. + + result @@ -6690,6 +7362,8 @@ are limited. + + Expression @@ -6716,6 +7390,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH FromUTCTimestamp @@ -6742,6 +7418,8 @@ are limited. + + timezone @@ -6763,6 +7441,8 @@ are limited. + + result @@ -6784,6 +7464,8 @@ are limited. + + FromUnixTime @@ -6810,6 +7492,8 @@ are limited. + + format @@ -6831,6 +7515,8 @@ are limited. + + result @@ -6852,6 +7538,8 @@ are limited. + + GetArrayItem @@ -6874,7 +7562,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -6899,6 +7589,8 @@ are limited. + + result @@ -6916,9 +7608,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -6942,7 +7636,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -6963,7 +7659,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -6993,6 +7691,8 @@ are limited. + + path @@ -7014,6 +7714,8 @@ are limited. + + result @@ -7035,6 +7737,8 @@ are limited. + + GetMapValue @@ -7058,7 +7762,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -7082,6 +7788,8 @@ are limited. NS NS NS +NS +NS result @@ -7099,9 +7807,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -7129,6 +7839,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH GetStructField @@ -7153,7 +7865,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -7172,9 +7886,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -7202,6 +7918,8 @@ are limited. + + format @@ -7223,6 +7941,8 @@ are limited. + + result @@ -7244,6 +7964,8 @@ are limited. + + GreaterThan @@ -7270,6 +7992,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + rhs @@ -7291,6 +8015,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + result @@ -7312,6 +8038,8 @@ are limited. + + AST @@ -7334,6 +8062,8 @@ are limited. NS NS + + rhs @@ -7355,6 +8085,8 @@ are limited. NS NS + + result @@ -7376,6 +8108,8 @@ are limited. + + GreaterThanOrEqual @@ -7402,6 +8136,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + rhs @@ -7423,6 +8159,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + result @@ -7444,6 +8182,8 @@ are limited. + + AST @@ -7466,6 +8206,8 @@ are limited. NS NS + + rhs @@ -7487,6 +8229,8 @@ are limited. NS NS + + result @@ -7508,6 +8252,8 @@ are limited. + + Expression @@ -7534,6 +8280,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Greatest @@ -7560,6 +8308,8 @@ are limited. NS NS + + result @@ -7581,6 +8331,8 @@ are limited. NS NS + + HiveHash @@ -7607,6 +8359,8 @@ are limited. NS NS NS +NS +NS result @@ -7628,6 +8382,8 @@ are limited. + + Hour @@ -7654,6 +8410,8 @@ are limited. + + result @@ -7675,6 +8433,8 @@ are limited. + + Hypot @@ -7701,6 +8461,8 @@ are limited. + + rhs @@ -7722,6 +8484,8 @@ are limited. + + result @@ -7743,6 +8507,8 @@ are limited. + + If @@ -7769,6 +8535,8 @@ are limited. + + trueValue @@ -7786,9 +8554,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -7807,9 +8577,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -7828,9 +8600,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -7858,6 +8632,8 @@ are limited. NS NS + + list @@ -7879,6 +8655,8 @@ are limited. NS NS + + result @@ -7900,6 +8678,8 @@ are limited. + + Expression @@ -7926,6 +8706,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH InSet @@ -7952,6 +8734,8 @@ are limited. NS NS + + result @@ -7973,6 +8757,8 @@ are limited. + + InitCap @@ -7999,6 +8785,8 @@ are limited. + + result @@ -8020,6 +8808,8 @@ are limited. + + InputFileBlockLength @@ -8046,6 +8836,8 @@ are limited. + + InputFileBlockStart @@ -8072,6 +8864,8 @@ are limited. + + InputFileName @@ -8098,6 +8892,8 @@ are limited. + + IntegralDivide @@ -8124,6 +8920,8 @@ are limited. + + rhs @@ -8145,6 +8943,8 @@ are limited. + + result @@ -8166,6 +8966,8 @@ are limited. + + IsNaN @@ -8192,6 +8994,8 @@ are limited. + + result @@ -8213,6 +9017,8 @@ are limited. + + IsNotNull @@ -8235,9 +9041,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -8260,6 +9068,8 @@ are limited. + + IsNull @@ -8282,9 +9092,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -8307,6 +9119,8 @@ are limited. + + Expression @@ -8333,6 +9147,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH JsonToStructs @@ -8359,6 +9175,8 @@ are limited. + + result @@ -8377,8 +9195,10 @@ are limited. NS -PS
MAP only supports keys and values that are of STRING type;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, MAP, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, MAP, UDT +PS
MAP only supports keys and values that are of STRING type;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH + + @@ -8406,6 +9226,8 @@ are limited. + + field @@ -8427,6 +9249,8 @@ are limited. + + result @@ -8448,6 +9272,8 @@ are limited. + + KnownFloatingPointNormalized @@ -8474,6 +9300,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S result @@ -8495,6 +9323,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S KnownNotNull @@ -8517,9 +9347,11 @@ are limited. NS S S -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -8538,9 +9370,11 @@ are limited. NS S S -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -8564,9 +9398,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT NS @@ -8589,6 +9425,8 @@ are limited. + + default @@ -8606,9 +9444,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT NS @@ -8627,9 +9467,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT NS @@ -8653,9 +9495,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -8674,9 +9518,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -8695,9 +9541,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -8725,6 +9573,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH LastDay @@ -8751,6 +9601,8 @@ are limited. + + result @@ -8772,6 +9624,8 @@ are limited. + + Lead @@ -8794,9 +9648,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT NS @@ -8819,6 +9675,8 @@ are limited. + + default @@ -8836,9 +9694,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT NS @@ -8857,9 +9717,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT NS @@ -8887,6 +9749,8 @@ are limited. NS NS + + result @@ -8908,6 +9772,8 @@ are limited. NS NS + + Length @@ -8934,6 +9800,8 @@ are limited. + + result @@ -8955,6 +9823,8 @@ are limited. + + LessThan @@ -8981,6 +9851,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + rhs @@ -9002,6 +9874,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + result @@ -9023,6 +9897,8 @@ are limited. + + AST @@ -9045,6 +9921,8 @@ are limited. NS NS + + rhs @@ -9066,6 +9944,8 @@ are limited. NS NS + + result @@ -9087,6 +9967,8 @@ are limited. + + Expression @@ -9113,6 +9995,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH LessThanOrEqual @@ -9139,6 +10023,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + rhs @@ -9160,6 +10046,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + result @@ -9181,6 +10069,8 @@ are limited. + + AST @@ -9203,6 +10093,8 @@ are limited. NS NS + + rhs @@ -9224,6 +10116,8 @@ are limited. NS NS + + result @@ -9245,6 +10139,8 @@ are limited. + + Like @@ -9271,6 +10167,8 @@ are limited. + + search @@ -9292,6 +10190,8 @@ are limited. + + result @@ -9313,6 +10213,8 @@ are limited. + + Literal @@ -9335,10 +10237,12 @@ are limited. S S S -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH NS +S +S AST @@ -9361,6 +10265,8 @@ are limited. NS NS NS +NS +NS Log @@ -9387,6 +10293,8 @@ are limited. + + result @@ -9408,6 +10316,8 @@ are limited. + + Log10 @@ -9434,6 +10344,8 @@ are limited. + + result @@ -9455,6 +10367,8 @@ are limited. + + Expression @@ -9481,6 +10395,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Log1p @@ -9507,6 +10423,8 @@ are limited. + + result @@ -9528,6 +10446,8 @@ are limited. + + Log2 @@ -9554,6 +10474,8 @@ are limited. + + result @@ -9575,6 +10497,8 @@ are limited. + + Logarithm @@ -9601,6 +10525,8 @@ are limited. + + base @@ -9622,6 +10548,8 @@ are limited. + + result @@ -9643,6 +10571,8 @@ are limited. + + Lower @@ -9669,6 +10599,8 @@ are limited. + + result @@ -9690,6 +10622,8 @@ are limited. + + MakeDecimal @@ -9716,6 +10650,8 @@ are limited. + + result @@ -9737,6 +10673,8 @@ are limited. + + MapConcat @@ -9760,7 +10698,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -9781,7 +10721,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -9807,7 +10749,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -9827,7 +10771,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -9857,6 +10803,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH MapFilter @@ -9880,7 +10828,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -9904,6 +10854,8 @@ are limited. + + result @@ -9922,7 +10874,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -9948,7 +10902,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -9968,7 +10924,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -9995,7 +10953,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -10015,7 +10975,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -10045,6 +11007,8 @@ are limited. + + result @@ -10066,6 +11030,8 @@ are limited. + + MicrosToTimestamp @@ -10092,6 +11058,8 @@ are limited. + + result @@ -10113,6 +11081,8 @@ are limited. + + MillisToTimestamp @@ -10139,6 +11109,8 @@ are limited. + + result @@ -10160,6 +11132,8 @@ are limited. + + Minute @@ -10186,6 +11160,8 @@ are limited. + + result @@ -10207,6 +11183,8 @@ are limited. + + Expression @@ -10233,6 +11211,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH MonotonicallyIncreasingID @@ -10259,6 +11239,8 @@ are limited. + + Month @@ -10285,6 +11267,8 @@ are limited. + + result @@ -10306,6 +11290,8 @@ are limited. + + Multiply @@ -10332,6 +11318,8 @@ are limited. + + rhs @@ -10353,6 +11341,8 @@ are limited. + + result @@ -10374,6 +11364,8 @@ are limited. + + AST @@ -10396,6 +11388,8 @@ are limited. + + rhs @@ -10417,6 +11411,8 @@ are limited. + + result @@ -10438,6 +11434,8 @@ are limited. + + Murmur3Hash @@ -10460,9 +11458,11 @@ are limited. S NS NS -PS
Arrays of structs are not supported;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
Arrays of structs are not supported;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT NS @@ -10485,6 +11485,8 @@ are limited. + + NaNvl @@ -10511,6 +11513,8 @@ are limited. + + rhs @@ -10532,6 +11536,8 @@ are limited. + + result @@ -10553,6 +11559,8 @@ are limited. + + NamedLambdaVariable @@ -10575,9 +11583,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -10605,6 +11615,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Not @@ -10631,6 +11643,8 @@ are limited. + + result @@ -10652,6 +11666,8 @@ are limited. + + AST @@ -10674,6 +11690,8 @@ are limited. + + result @@ -10695,6 +11713,8 @@ are limited. + + NthValue @@ -10717,9 +11737,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -10742,6 +11764,8 @@ are limited. + + result @@ -10759,9 +11783,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -10789,6 +11815,8 @@ are limited. + + result @@ -10810,6 +11838,8 @@ are limited. + + Or @@ -10836,6 +11866,8 @@ are limited. + + rhs @@ -10857,6 +11889,8 @@ are limited. + + result @@ -10878,6 +11912,8 @@ are limited. + + AST @@ -10900,6 +11936,8 @@ are limited. + + rhs @@ -10921,6 +11959,8 @@ are limited. + + result @@ -10942,6 +11982,8 @@ are limited. + + Expression @@ -10968,6 +12010,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH ParseUrl @@ -10994,6 +12038,8 @@ are limited. + + partToExtract @@ -11015,6 +12061,8 @@ are limited. + + key @@ -11036,6 +12084,8 @@ are limited. + + result @@ -11057,6 +12107,8 @@ are limited. + + PercentRank @@ -11083,6 +12135,8 @@ are limited. NS NS NS +NS +NS result @@ -11104,6 +12158,8 @@ are limited. + + Pmod @@ -11130,6 +12186,8 @@ are limited. + + rhs @@ -11151,6 +12209,8 @@ are limited. + + result @@ -11172,6 +12232,8 @@ are limited. + + PosExplode @@ -11194,8 +12256,10 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -11215,7 +12279,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -11245,6 +12311,8 @@ are limited. + + rhs @@ -11266,6 +12334,8 @@ are limited. + + result @@ -11287,6 +12357,8 @@ are limited. + + AST @@ -11309,6 +12381,8 @@ are limited. + + rhs @@ -11330,6 +12404,8 @@ are limited. + + result @@ -11351,6 +12427,8 @@ are limited. + + Expression @@ -11377,6 +12455,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH PreciseTimestampConversion @@ -11403,6 +12483,8 @@ are limited. + + result @@ -11424,6 +12506,8 @@ are limited. + + PromotePrecision @@ -11450,6 +12534,8 @@ are limited. + + result @@ -11471,6 +12557,8 @@ are limited. + + PythonUDF @@ -11493,9 +12581,11 @@ are limited. NS NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT NS @@ -11518,6 +12608,8 @@ are limited. NS PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, MAP + + reduction @@ -11536,9 +12628,11 @@ are limited. NS NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT NS @@ -11561,6 +12655,8 @@ are limited. NS PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, MAP + + window @@ -11579,9 +12675,11 @@ are limited. NS NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT NS @@ -11604,6 +12702,8 @@ are limited. NS PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, MAP + + project @@ -11622,9 +12722,11 @@ are limited. NS NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT NS @@ -11647,6 +12749,8 @@ are limited. NS PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types DECIMAL, NULL, BINARY, MAP + + Quarter @@ -11673,6 +12777,8 @@ are limited. + + result @@ -11694,10 +12800,12 @@ are limited. + + RLike -`rlike` +`regexp_like`, `regexp`, `rlike` Regular expression version of Like None project @@ -11720,6 +12828,8 @@ are limited. + + regexp @@ -11741,6 +12851,8 @@ are limited. + + result @@ -11762,6 +12874,8 @@ are limited. + + Expression @@ -11788,6 +12902,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH RaiseError @@ -11814,6 +12930,8 @@ are limited. + + result @@ -11835,6 +12953,8 @@ are limited. + + Rand @@ -11861,6 +12981,8 @@ are limited. + + result @@ -11882,6 +13004,8 @@ are limited. + + Rank @@ -11908,6 +13032,8 @@ are limited. NS NS NS +NS +NS result @@ -11929,6 +13055,8 @@ are limited. + + RegExpExtract @@ -11955,6 +13083,8 @@ are limited. + + regexp @@ -11976,6 +13106,8 @@ are limited. + + idx @@ -11997,6 +13129,8 @@ are limited. + + result @@ -12018,6 +13152,8 @@ are limited. + + RegExpExtractAll @@ -12044,6 +13180,8 @@ are limited. + + regexp @@ -12065,6 +13203,8 @@ are limited. + + idx @@ -12086,6 +13226,8 @@ are limited. + + result @@ -12107,6 +13249,8 @@ are limited. + + RegExpReplace @@ -12133,6 +13277,8 @@ are limited. + + result @@ -12154,6 +13300,8 @@ are limited. + + pos @@ -12175,6 +13323,8 @@ are limited. + + str @@ -12196,6 +13346,8 @@ are limited. + + rep @@ -12217,6 +13369,8 @@ are limited. + + Expression @@ -12243,6 +13397,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Remainder @@ -12269,6 +13425,8 @@ are limited. + + rhs @@ -12290,6 +13448,8 @@ are limited. + + result @@ -12311,6 +13471,8 @@ are limited. + + ReplicateRows @@ -12333,9 +13495,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT NS @@ -12354,7 +13518,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH + + @@ -12384,6 +13550,8 @@ are limited. + + result @@ -12405,6 +13573,8 @@ are limited. + + Rint @@ -12431,6 +13601,8 @@ are limited. + + result @@ -12452,6 +13624,8 @@ are limited. + + AST @@ -12474,6 +13648,8 @@ are limited. + + result @@ -12495,6 +13671,8 @@ are limited. + + Round @@ -12521,6 +13699,8 @@ are limited. + + scale @@ -12542,6 +13722,8 @@ are limited. + + result @@ -12563,6 +13745,8 @@ are limited. + + RowNumber @@ -12589,6 +13773,8 @@ are limited. + + Expression @@ -12615,6 +13801,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH ScalaUDF @@ -12637,9 +13825,11 @@ are limited. S S S -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -12658,9 +13848,11 @@ are limited. S S S -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -12688,6 +13880,8 @@ are limited. + + result @@ -12709,6 +13903,8 @@ are limited. + + SecondsToTimestamp @@ -12735,6 +13931,8 @@ are limited. + + result @@ -12756,6 +13954,8 @@ are limited. + + Sequence @@ -12782,6 +13982,8 @@ are limited. + + stop @@ -12803,6 +14005,8 @@ are limited. + + step @@ -12824,6 +14028,8 @@ are limited. + + result @@ -12845,6 +14051,8 @@ are limited. + + ShiftLeft @@ -12871,6 +14079,8 @@ are limited. + + amount @@ -12892,6 +14102,8 @@ are limited. + + result @@ -12913,6 +14125,8 @@ are limited. + + ShiftRight @@ -12939,6 +14153,8 @@ are limited. + + amount @@ -12960,6 +14176,8 @@ are limited. + + result @@ -12981,6 +14199,8 @@ are limited. + + Expression @@ -13007,6 +14227,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH ShiftRightUnsigned @@ -13033,6 +14255,8 @@ are limited. + + amount @@ -13054,6 +14278,8 @@ are limited. + + result @@ -13075,6 +14301,8 @@ are limited. + + Signum @@ -13089,7 +14317,9 @@ are limited. -S +S + + @@ -13122,6 +14352,8 @@ are limited. + + Sin @@ -13148,6 +14380,8 @@ are limited. + + result @@ -13169,6 +14403,8 @@ are limited. + + AST @@ -13191,6 +14427,8 @@ are limited. + + result @@ -13212,6 +14450,8 @@ are limited. + + Sinh @@ -13238,6 +14478,8 @@ are limited. + + result @@ -13259,6 +14501,8 @@ are limited. + + AST @@ -13281,6 +14525,8 @@ are limited. + + result @@ -13302,6 +14548,8 @@ are limited. + + Size @@ -13324,8 +14572,10 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -13349,6 +14599,8 @@ are limited. + + Expression @@ -13375,6 +14627,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH SortArray @@ -13397,7 +14651,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, UDT, DAYTIME, YEARMONTH + + @@ -13422,6 +14678,8 @@ are limited. + + result @@ -13439,7 +14697,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, UDT, DAYTIME, YEARMONTH + + @@ -13469,6 +14729,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + result @@ -13490,6 +14752,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + SparkPartitionID @@ -13516,6 +14780,8 @@ are limited. + + SpecifiedWindowFrame @@ -13542,6 +14808,8 @@ are limited. +S +NS upper @@ -13563,6 +14831,8 @@ are limited. +S +NS result @@ -13584,6 +14854,8 @@ are limited. +S +NS Sqrt @@ -13610,6 +14882,8 @@ are limited. + + result @@ -13631,6 +14905,8 @@ are limited. + + AST @@ -13653,6 +14929,8 @@ are limited. + + result @@ -13674,6 +14952,8 @@ are limited. + + Stack @@ -13700,6 +14980,8 @@ are limited. + + expr @@ -13717,9 +14999,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -13738,7 +15022,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -13768,6 +15054,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH StartsWith @@ -13794,6 +15082,8 @@ are limited. + + search @@ -13815,6 +15105,8 @@ are limited. + + result @@ -13836,6 +15128,8 @@ are limited. + + StringInstr @@ -13862,6 +15156,8 @@ are limited. + + substr @@ -13883,6 +15179,8 @@ are limited. + + result @@ -13904,6 +15202,8 @@ are limited. + + StringLPad @@ -13930,6 +15230,8 @@ are limited. + + len @@ -13951,6 +15253,8 @@ are limited. + + pad @@ -13972,6 +15276,8 @@ are limited. + + result @@ -13993,6 +15299,8 @@ are limited. + + StringLocate @@ -14019,6 +15327,8 @@ are limited. + + str @@ -14040,6 +15350,8 @@ are limited. + + start @@ -14061,6 +15373,8 @@ are limited. + + result @@ -14082,6 +15396,8 @@ are limited. + + StringRPad @@ -14108,6 +15424,8 @@ are limited. + + len @@ -14129,6 +15447,8 @@ are limited. + + pad @@ -14150,6 +15470,8 @@ are limited. + + result @@ -14171,6 +15493,8 @@ are limited. + + Expression @@ -14197,6 +15521,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH StringRepeat @@ -14223,6 +15549,8 @@ are limited. + + repeatTimes @@ -14244,6 +15572,8 @@ are limited. + + result @@ -14265,6 +15595,8 @@ are limited. + + StringReplace @@ -14291,6 +15623,8 @@ are limited. + + search @@ -14312,6 +15646,8 @@ are limited. + + replace @@ -14333,6 +15669,8 @@ are limited. + + result @@ -14354,6 +15692,8 @@ are limited. + + StringSplit @@ -14380,6 +15720,8 @@ are limited. + + regexp @@ -14401,6 +15743,8 @@ are limited. + + limit @@ -14422,6 +15766,8 @@ are limited. + + result @@ -14443,6 +15789,8 @@ are limited. + + StringToMap @@ -14469,6 +15817,8 @@ are limited. + + pairDelim @@ -14490,6 +15840,8 @@ are limited. + + keyValueDelim @@ -14511,6 +15863,8 @@ are limited. + + result @@ -14532,6 +15886,8 @@ are limited. S + + Expression @@ -14558,6 +15914,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH StringTranslate @@ -14584,6 +15942,8 @@ are limited. + + from @@ -14605,6 +15965,8 @@ are limited. + + to @@ -14626,6 +15988,8 @@ are limited. + + result @@ -14647,6 +16011,8 @@ are limited. + + StringTrim @@ -14673,6 +16039,8 @@ are limited. + + trimStr @@ -14694,6 +16062,8 @@ are limited. + + result @@ -14715,6 +16085,8 @@ are limited. + + StringTrimLeft @@ -14741,6 +16113,8 @@ are limited. + + trimStr @@ -14762,6 +16136,8 @@ are limited. + + result @@ -14783,6 +16159,8 @@ are limited. + + StringTrimRight @@ -14809,6 +16187,8 @@ are limited. + + trimStr @@ -14830,6 +16210,8 @@ are limited. + + result @@ -14851,6 +16233,8 @@ are limited. + + StructsToJson @@ -14877,6 +16261,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP + + result @@ -14898,6 +16284,8 @@ are limited. + + Expression @@ -14924,6 +16312,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Substring @@ -14950,6 +16340,8 @@ are limited. + + pos @@ -14971,6 +16363,8 @@ are limited. + + len @@ -14992,6 +16386,8 @@ are limited. + + result @@ -15013,6 +16409,8 @@ are limited. + + SubstringIndex @@ -15039,6 +16437,8 @@ are limited. + + delim @@ -15060,6 +16460,8 @@ are limited. + + count @@ -15081,6 +16483,8 @@ are limited. + + result @@ -15102,6 +16506,8 @@ are limited. + + Subtract @@ -15128,6 +16534,8 @@ are limited. +NS +NS rhs @@ -15149,6 +16557,8 @@ are limited. +NS +NS result @@ -15170,6 +16580,8 @@ are limited. +NS +NS AST @@ -15192,6 +16604,8 @@ are limited. +NS +NS rhs @@ -15213,6 +16627,8 @@ are limited. +NS +NS result @@ -15234,6 +16650,8 @@ are limited. +NS +NS Tan @@ -15260,6 +16678,8 @@ are limited. + + result @@ -15281,6 +16701,8 @@ are limited. + + AST @@ -15303,6 +16725,8 @@ are limited. + + result @@ -15324,6 +16748,8 @@ are limited. + + Expression @@ -15350,6 +16776,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Tanh @@ -15376,6 +16804,8 @@ are limited. + + result @@ -15397,6 +16827,8 @@ are limited. + + AST @@ -15419,6 +16851,8 @@ are limited. + + result @@ -15440,6 +16874,8 @@ are limited. + + TimeAdd @@ -15466,6 +16902,8 @@ are limited. + + interval @@ -15482,10 +16920,12 @@ are limited. -PS
month intervals are not supported;
Literal value only +PS
Literal value only + +PS
Literal value only @@ -15508,6 +16948,8 @@ are limited. + + ToDegrees @@ -15534,6 +16976,8 @@ are limited. + + result @@ -15555,6 +16999,8 @@ are limited. + + ToRadians @@ -15581,6 +17027,8 @@ are limited. + + result @@ -15602,6 +17050,8 @@ are limited. + + ToUTCTimestamp @@ -15628,6 +17078,8 @@ are limited. + + timezone @@ -15649,6 +17101,8 @@ are limited. + + result @@ -15670,6 +17124,8 @@ are limited. + + ToUnixTimestamp @@ -15696,6 +17152,8 @@ are limited. + + format @@ -15717,6 +17175,8 @@ are limited. + + result @@ -15738,6 +17198,8 @@ are limited. + + Expression @@ -15764,6 +17226,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH TransformKeys @@ -15787,7 +17251,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -15811,6 +17277,8 @@ are limited. NS NS +NS +NS result @@ -15829,7 +17297,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -15855,7 +17325,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -15875,9 +17347,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -15897,7 +17371,9 @@ are limited. -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -15926,6 +17402,8 @@ are limited. +NS +NS result @@ -15947,6 +17425,8 @@ are limited. +NS +NS AST @@ -15969,6 +17449,8 @@ are limited. +NS +NS result @@ -15990,6 +17472,8 @@ are limited. +NS +NS UnaryPositive @@ -16016,6 +17500,8 @@ are limited. +NS +NS result @@ -16037,6 +17523,8 @@ are limited. +NS +NS AST @@ -16059,6 +17547,8 @@ are limited. +NS +NS result @@ -16080,6 +17570,8 @@ are limited. +NS +NS UnboundedFollowing$ @@ -16106,6 +17598,8 @@ are limited. + + Expression @@ -16132,6 +17626,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH UnboundedPreceding$ @@ -16158,6 +17654,8 @@ are limited. + + UnixTimestamp @@ -16184,6 +17682,8 @@ are limited. + + format @@ -16205,6 +17705,8 @@ are limited. + + result @@ -16226,6 +17728,8 @@ are limited. + + UnscaledValue @@ -16252,6 +17756,8 @@ are limited. + + result @@ -16273,6 +17779,8 @@ are limited. + + Upper @@ -16299,6 +17807,8 @@ are limited. + + result @@ -16320,6 +17830,8 @@ are limited. + + WeekDay @@ -16346,6 +17858,8 @@ are limited. + + result @@ -16367,6 +17881,8 @@ are limited. + + WindowExpression @@ -16393,6 +17909,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S windowSpec @@ -16414,6 +17932,8 @@ are limited. +S +NS result @@ -16435,6 +17955,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S WindowSpecDefinition @@ -16459,7 +17981,9 @@ are limited. NS NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -16480,7 +18004,9 @@ are limited. NS NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -16501,7 +18027,9 @@ are limited. NS NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -16529,6 +18057,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH XxHash64 @@ -16542,8 +18072,8 @@ are limited. S S S -NS -NS +S +S S PS
UTC is only supported TZ for TIMESTAMP S @@ -16555,6 +18085,8 @@ are limited. NS NS NS +NS +NS result @@ -16576,6 +18108,8 @@ are limited. + + Year @@ -16602,6 +18136,8 @@ are limited. + + result @@ -16623,6 +18159,8 @@ are limited. + + AggregateExpression @@ -16649,6 +18187,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S filter @@ -16670,6 +18210,8 @@ are limited. + + result @@ -16691,6 +18233,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S reduction @@ -16713,6 +18257,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S filter @@ -16734,6 +18280,8 @@ are limited. + + result @@ -16755,6 +18303,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S window @@ -16777,6 +18327,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S filter @@ -16798,6 +18350,8 @@ are limited. + + result @@ -16819,6 +18373,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S ApproximatePercentile @@ -16845,6 +18401,8 @@ are limited. + + percentage @@ -16866,6 +18424,8 @@ are limited. + + accuracy @@ -16887,6 +18447,8 @@ are limited. + + result @@ -16908,6 +18470,8 @@ are limited. + + reduction @@ -16930,6 +18494,8 @@ are limited. + + percentage @@ -16951,6 +18517,8 @@ are limited. + + accuracy @@ -16972,6 +18540,8 @@ are limited. + + result @@ -16993,6 +18563,8 @@ are limited. + + Expression @@ -17019,6 +18591,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Average @@ -17038,13 +18612,15 @@ are limited. S +S +NS - - +NS +NS result @@ -17066,6 +18642,8 @@ are limited. + + reduction @@ -17081,13 +18659,15 @@ are limited. S +S +NS - - +NS +NS result @@ -17109,6 +18689,8 @@ are limited. + + window @@ -17124,13 +18706,15 @@ are limited. S +S +NS - - +NS +NS result @@ -17152,6 +18736,8 @@ are limited. + + CollectList @@ -17174,9 +18760,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17195,7 +18783,9 @@ are limited. -PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -17217,9 +18807,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17238,7 +18830,9 @@ are limited. -PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -17260,9 +18854,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17281,7 +18877,9 @@ are limited. -PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH + + @@ -17307,9 +18905,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT NS @@ -17328,7 +18928,9 @@ are limited. -PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH + + @@ -17350,9 +18952,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT NS @@ -17371,7 +18975,9 @@ are limited. -PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH + + @@ -17393,9 +18999,11 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT NS @@ -17414,7 +19022,9 @@ are limited. -PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
window operations are disabled by default due to extreme memory usage;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH + + @@ -17444,6 +19054,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Count @@ -17470,6 +19082,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S result @@ -17491,6 +19105,8 @@ are limited. + + reduction @@ -17513,6 +19129,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S result @@ -17534,6 +19152,8 @@ are limited. + + window @@ -17556,6 +19176,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP PS
UTC is only supported TZ for child TIMESTAMP S +S +S result @@ -17577,6 +19199,8 @@ are limited. + + First @@ -17599,9 +19223,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17620,9 +19246,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17642,9 +19270,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17663,9 +19293,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17685,9 +19317,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17706,9 +19340,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17732,9 +19368,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17753,9 +19391,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17775,9 +19415,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17796,9 +19438,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17818,9 +19462,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17839,9 +19485,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -17869,6 +19517,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Max @@ -17895,6 +19545,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT NS + + result @@ -17916,6 +19568,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT NS + + reduction @@ -17938,6 +19592,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT NS + + result @@ -17959,6 +19615,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT NS + + window @@ -17981,6 +19639,8 @@ are limited. NS NS + + result @@ -18002,6 +19662,8 @@ are limited. NS NS + + Min @@ -18028,6 +19690,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT NS + + result @@ -18049,6 +19713,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT NS + + reduction @@ -18071,6 +19737,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT NS + + result @@ -18092,6 +19760,8 @@ are limited. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT NS + + window @@ -18114,6 +19784,8 @@ are limited. NS NS + + result @@ -18135,6 +19807,8 @@ are limited. NS NS + + Percentile @@ -18161,6 +19835,8 @@ are limited. + + percentage @@ -18182,6 +19858,8 @@ are limited. + + frequency @@ -18203,6 +19881,8 @@ are limited. + + result @@ -18224,6 +19904,8 @@ are limited. + + reduction @@ -18246,6 +19928,8 @@ are limited. + + percentage @@ -18267,6 +19951,8 @@ are limited. + + frequency @@ -18288,6 +19974,8 @@ are limited. + + result @@ -18309,6 +19997,8 @@ are limited. + + Expression @@ -18335,6 +20025,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH PivotFirst @@ -18361,6 +20053,8 @@ are limited. NS NS NS +NS +NS valueColumn @@ -18382,6 +20076,8 @@ are limited. NS NS NS +NS +NS result @@ -18399,7 +20095,9 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH +NS +NS NS NS NS @@ -18425,6 +20123,8 @@ are limited. NS NS NS +NS +NS valueColumn @@ -18446,6 +20146,8 @@ are limited. NS NS NS +NS +NS result @@ -18463,7 +20165,9 @@ are limited. S NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types NULL, BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT, DAYTIME, YEARMONTH +NS +NS NS NS NS @@ -18493,6 +20197,8 @@ are limited. + + result @@ -18514,6 +20220,8 @@ are limited. + + aggregation @@ -18536,6 +20244,8 @@ are limited. + + result @@ -18557,6 +20267,8 @@ are limited. + + window @@ -18579,6 +20291,8 @@ are limited. + + result @@ -18600,6 +20314,8 @@ are limited. + + StddevSamp @@ -18626,6 +20342,8 @@ are limited. + + result @@ -18647,6 +20365,8 @@ are limited. + + reduction @@ -18669,6 +20389,8 @@ are limited. + + result @@ -18690,6 +20412,8 @@ are limited. + + window @@ -18712,6 +20436,8 @@ are limited. + + result @@ -18733,6 +20459,8 @@ are limited. + + Expression @@ -18759,6 +20487,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH Sum @@ -18785,6 +20515,8 @@ are limited. + + result @@ -18806,6 +20538,8 @@ are limited. + + reduction @@ -18828,6 +20562,8 @@ are limited. + + result @@ -18849,6 +20585,8 @@ are limited. + + window @@ -18871,6 +20609,8 @@ are limited. + + result @@ -18892,6 +20632,8 @@ are limited. + + VariancePop @@ -18918,6 +20660,8 @@ are limited. + + result @@ -18939,6 +20683,8 @@ are limited. + + aggregation @@ -18961,6 +20707,8 @@ are limited. + + result @@ -18982,6 +20730,8 @@ are limited. + + window @@ -19004,6 +20754,8 @@ are limited. + + result @@ -19025,6 +20777,8 @@ are limited. + + VarianceSamp @@ -19051,6 +20805,8 @@ are limited. + + result @@ -19072,6 +20828,8 @@ are limited. + + aggregation @@ -19094,6 +20852,8 @@ are limited. + + result @@ -19115,6 +20875,8 @@ are limited. + + window @@ -19137,6 +20899,8 @@ are limited. + + result @@ -19158,6 +20922,8 @@ are limited. + + Expression @@ -19184,6 +20950,8 @@ are limited. MAP STRUCT UDT +DAYTIME +YEARMONTH NormalizeNaNAndZero @@ -19210,6 +20978,8 @@ are limited. + + result @@ -19231,6 +21001,8 @@ are limited. + + ScalarSubquery @@ -19253,9 +21025,11 @@ are limited. S S NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -19279,9 +21053,11 @@ are limited. S S S -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -19300,9 +21076,11 @@ are limited. S S S -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -19326,9 +21104,11 @@ are limited. S S S -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -19347,9 +21127,11 @@ are limited. S S S -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT, DAYTIME, YEARMONTH +NS +NS NS @@ -19371,7 +21153,7 @@ and the accelerator produces the same result. ### `AnsiCast` - + @@ -19391,8 +21173,10 @@ and the accelerator produces the same result. + + - + @@ -19404,7 +21188,9 @@ and the accelerator produces the same result. - + + + @@ -19433,6 +21219,8 @@ and the accelerator produces the same result. + + @@ -19454,6 +21242,8 @@ and the accelerator produces the same result. + + @@ -19475,6 +21265,8 @@ and the accelerator produces the same result. + + @@ -19496,6 +21288,8 @@ and the accelerator produces the same result. + + @@ -19517,6 +21311,8 @@ and the accelerator produces the same result. + + @@ -19538,6 +21334,8 @@ and the accelerator produces the same result. + + @@ -19559,6 +21357,8 @@ and the accelerator produces the same result. + + @@ -19580,6 +21380,8 @@ and the accelerator produces the same result. + + @@ -19590,8 +21392,8 @@ and the accelerator produces the same result. - - + + @@ -19601,6 +21403,8 @@ and the accelerator produces the same result. + + @@ -19622,6 +21426,8 @@ and the accelerator produces the same result. + + @@ -19643,6 +21449,8 @@ and the accelerator produces the same result. + + @@ -19664,6 +21472,8 @@ and the accelerator produces the same result. + + @@ -19685,6 +21495,8 @@ and the accelerator produces the same result. + + @@ -19702,7 +21514,9 @@ and the accelerator produces the same result. - + + + @@ -19724,7 +21538,9 @@ and the accelerator produces the same result. - + + + @@ -19746,7 +21562,9 @@ and the accelerator produces the same result. - + + + @@ -19769,13 +21587,61 @@ and the accelerator produces the same result. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TO
TO
BOOLEAN BYTEMAP STRUCT UDTDAYTIMEYEARMONTH
FROM
FROM BOOLEAN S S SSNS
SHORT
INT
LONG
FLOAT
DOUBLE
DATE
TIMESTAMP
STRINGS S SSPS
UTC is only supported TZ for TIMESTAMP		PS
Only 4 digit year parsing is available. To enable parsing anyways set spark.rapids.sql.hasExtendedYearValues to false.		PS
Only 4 digit year parsing is available. To enable parsing anyways set spark.rapids.sql.hasExtendedYearValues to false.;
UTC is only supported TZ for TIMESTAMP		 S S
DECIMAL
NULLNS NS NSNSNS
BINARY
CALENDAR
ARRAY PS
The array's child type must also support being cast to the desired child type;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, MAP, UDT		PS
The array's child type must also support being cast to the desired child type;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, MAP, UDT, DAYTIME, YEARMONTH		 PS
the map's key and value must also support being cast to the desired child types;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT		PS
the map's key and value must also support being cast to the desired child types;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH
PS
the struct's children must also support being cast to the desired child type(s);
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, MAP, UDT		PS
the struct's children must also support being cast to the desired child type(s);
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, MAP, UDT, DAYTIME, YEARMONTH
NS
DAYTIME NS NS
YEARMONTH NS NS
### `Cast` - + @@ -19795,8 +21661,10 @@ and the accelerator produces the same result. + + - + @@ -19816,6 +21684,8 @@ and the accelerator produces the same result. + + @@ -19837,6 +21707,8 @@ and the accelerator produces the same result. + + @@ -19858,6 +21730,8 @@ and the accelerator produces the same result. + + @@ -19879,6 +21753,8 @@ and the accelerator produces the same result. + + @@ -19900,6 +21776,8 @@ and the accelerator produces the same result. + + @@ -19921,6 +21799,8 @@ and the accelerator produces the same result. + + @@ -19942,6 +21822,8 @@ and the accelerator produces the same result. + + @@ -19963,6 +21845,8 @@ and the accelerator produces the same result. + + @@ -19984,6 +21868,8 @@ and the accelerator produces the same result. + + @@ -20005,6 +21891,8 @@ and the accelerator produces the same result. + + @@ -20026,6 +21914,8 @@ and the accelerator produces the same result. + + @@ -20047,6 +21937,8 @@ and the accelerator produces the same result. + + @@ -20068,6 +21960,8 @@ and the accelerator produces the same result. + + @@ -20089,6 +21983,8 @@ and the accelerator produces the same result. + + @@ -20106,7 +22002,9 @@ and the accelerator produces the same result. - + + + @@ -20128,7 +22026,9 @@ and the accelerator produces the same result. - + + + @@ -20150,7 +22050,9 @@ and the accelerator produces the same result. - + + + @@ -20173,6 +22075,54 @@ and the accelerator produces the same result. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TO
TO
BOOLEAN BYTEMAP STRUCT UDTDAYTIMEYEARMONTH
FROM
FROM BOOLEAN S S
BYTE
SHORT
INT
LONG
FLOAT
DOUBLE
DATE
TIMESTAMP
STRING
DECIMAL
NULLNS NS NSNSNS
BINARY
CALENDAR
ARRAY PS
The array's child type must also support being cast to the desired child type(s);
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT		PS
The array's child type must also support being cast to the desired child type(s);
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH		 PS
the map's key and value must also support being cast to the desired child types;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT		PS
the map's key and value must also support being cast to the desired child types;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH
PS
the struct's children must also support being cast to the desired child type(s);
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT		PS
the struct's children must also support being cast to the desired child type(s);
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH
NS
DAYTIME NS NS
YEARMONTH NS NS
@@ -20209,6 +22159,8 @@ as `a` don't show up in the table. They are controlled by the rules for MAP STRUCT UDT +DAYTIME +YEARMONTH HashPartitioning @@ -20229,9 +22181,11 @@ as `a` don't show up in the table. They are controlled by the rules for S NS NS -PS
Arrays of structs are not supported;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT +PS
Arrays of structs are not supported;
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT, DAYTIME, YEARMONTH +NS NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, MAP, UDT NS @@ -20257,6 +22211,8 @@ as `a` don't show up in the table. They are controlled by the rules for PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, ARRAY, UDT NS + + RoundRobinPartitioning @@ -20281,6 +22237,8 @@ as `a` don't show up in the table. They are controlled by the rules for + + SinglePartition$ @@ -20305,6 +22263,8 @@ as `a` don't show up in the table. They are controlled by the rules for + + @@ -20335,6 +22295,8 @@ dates or timestamps, or for a lack of type coercion support. MAP STRUCT UDT +DAYTIME +YEARMONTH Avro @@ -20357,6 +22319,8 @@ dates or timestamps, or for a lack of type coercion support. NS NS NS + + Write @@ -20378,6 +22342,8 @@ dates or timestamps, or for a lack of type coercion support. NS NS NS + + CSV @@ -20400,6 +22366,8 @@ dates or timestamps, or for a lack of type coercion support. + + Write @@ -20421,6 +22389,8 @@ dates or timestamps, or for a lack of type coercion support. + + Delta @@ -20443,6 +22413,8 @@ dates or timestamps, or for a lack of type coercion support. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT NS + + Write @@ -20464,6 +22436,8 @@ dates or timestamps, or for a lack of type coercion support. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT NS + + HiveText @@ -20486,6 +22460,8 @@ dates or timestamps, or for a lack of type coercion support. NS NS NS +NS +NS Write @@ -20507,6 +22483,8 @@ dates or timestamps, or for a lack of type coercion support. NS NS NS +NS +NS Iceberg @@ -20529,6 +22507,8 @@ dates or timestamps, or for a lack of type coercion support. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT NS + + Write @@ -20550,6 +22530,8 @@ dates or timestamps, or for a lack of type coercion support. NS NS NS + + JSON @@ -20572,6 +22554,8 @@ dates or timestamps, or for a lack of type coercion support. NS PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, MAP, UDT NS + + Write @@ -20593,6 +22577,8 @@ dates or timestamps, or for a lack of type coercion support. NS NS NS + + ORC @@ -20615,6 +22601,8 @@ dates or timestamps, or for a lack of type coercion support. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, UDT PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, UDT NS + + Write @@ -20636,6 +22624,8 @@ dates or timestamps, or for a lack of type coercion support. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, MAP, UDT PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, MAP, UDT NS + + Parquet @@ -20658,6 +22648,8 @@ dates or timestamps, or for a lack of type coercion support. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT NS + + Write @@ -20679,6 +22671,8 @@ dates or timestamps, or for a lack of type coercion support. PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types UDT NS + + diff --git a/jenkins/hadoop-def.sh b/jenkins/hadoop-def.sh index 771705fc2ca..fcf1bf14faa 100755 --- a/jenkins/hadoop-def.sh +++ b/jenkins/hadoop-def.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ set -e -spark_version=${1:-"3.1.1"} +spark_version=${1:-"3.2.0"} scala_version=${2:-"2.12"} # Split spark version into base version (e.g. 3.3.0) and suffix (e.g. SNAPSHOT) PRE_IFS=$IFS diff --git a/jenkins/spark-nightly-build.sh b/jenkins/spark-nightly-build.sh index c5ef53da47d..3ec2e0f19b2 100755 --- a/jenkins/spark-nightly-build.sh +++ b/jenkins/spark-nightly-build.sh @@ -34,7 +34,7 @@ MVN="mvn -Dmaven.wagon.http.retryHandler.count=3 -DretryFailedDeploymentCount=3 DIST_PL="dist" function mvnEval { - $MVN help:evaluate -q -pl $DIST_PL $MVN_URM_MIRROR -Prelease311 -Dmaven.repo.local=$M2DIR -DforceStdout -Dexpression=$1 + $MVN help:evaluate -q -pl $DIST_PL $MVN_URM_MIRROR -Prelease320 -Dmaven.repo.local=$M2DIR -DforceStdout -Dexpression=$1 } ART_ID=$(mvnEval project.artifactId) @@ -176,7 +176,7 @@ distWithReducedPom "install" if [[ $SKIP_DEPLOY != 'true' ]]; then distWithReducedPom "deploy" - # this deploys selected submodules that is unconditionally built with Spark 3.1.1 + # this deploys selected submodules that is unconditionally built with Spark 3.2.0 $MVN -B deploy -pl $DEPLOY_SUBMODULES \ -Dbuildver=$SPARK_BASE_SHIM_VERSION \ -DskipTests \ diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh index e81db74cbd4..9cd1664af9f 100755 --- a/jenkins/spark-premerge-build.sh +++ b/jenkins/spark-premerge-build.sh @@ -83,16 +83,17 @@ mvn_verify() { # The jacoco coverage should have been collected, but because of how the shade plugin # works and jacoco we need to clean some things up so jacoco will only report for the # things we care about - SPK_VER=${JACOCO_SPARK_VER:-"311"} + SPK_VER=${JACOCO_SPARK_VER:-"320"} mkdir -p target/jacoco_classes/ FILE=$(ls dist/target/rapids-4-spark_2.12-*.jar | grep -v test | xargs readlink -f) UDF_JAR=$(ls ./udf-compiler/target/spark${SPK_VER}/rapids-4-spark-udf_2.12-*-spark${SPK_VER}.jar | grep -v test | xargs readlink -f) pushd target/jacoco_classes/ - jar xf $FILE com org rapids spark-shared "spark${JACOCO_SPARK_VER:-311}/" + jar xf $FILE com org rapids spark-shared "spark${JACOCO_SPARK_VER:-320}/" # extract the .class files in udf jar and replace the existing ones in spark3xx-ommon and spark$SPK_VER # because the class files in udf jar will be modified in aggregator's shade phase jar xf "$UDF_JAR" com/nvidia/spark/udf - rm -rf com/nvidia/shaded/ org/openucx/ spark-shared/com/nvidia/spark/udf/ spark${SPK_VER}/com/nvidia/spark/udf/ + # TODO Should clean up unused and duplicated 'org/roaringbitmap' in the spark3xx shim folders, https://github.com/NVIDIA/spark-rapids/issues/11175 + rm -rf com/nvidia/shaded/ org/openucx/ spark${SPK_VER}/META-INF/versions/*/org/roaringbitmap/ spark-shared/com/nvidia/spark/udf/ spark${SPK_VER}/com/nvidia/spark/udf/ popd # Triggering here until we change the jenkins file @@ -222,7 +223,7 @@ ci_scala213() { } prepare_spark() { - spark_ver=${1:-'3.1.1'} + spark_ver=${1:-'3.2.0'} scala_ver=${2:-'2.12'} ARTF_ROOT="$(pwd)/.download" diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh index 8d0ed32e8b8..71e580f38c5 100755 --- a/jenkins/spark-tests.sh +++ b/jenkins/spark-tests.sh @@ -59,7 +59,7 @@ $MVN_GET_CMD -DremoteRepositories=$PROJECT_TEST_REPO \ -DgroupId=com.nvidia -DartifactId=rapids-4-spark-integration-tests_$SCALA_BINARY_VER -Dversion=$PROJECT_TEST_VER -Dclassifier=pytest -Dpackaging=tar.gz RAPIDS_INT_TESTS_HOME="$ARTF_ROOT/integration_tests/" -# The version of pytest.tar.gz that is uploaded is the one built against spark311 but its being pushed without classifier for now +# The version of pytest.tar.gz that is uploaded is the one built against spark320 but its being pushed without classifier for now RAPIDS_INT_TESTS_TGZ="$ARTF_ROOT/rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-$PROJECT_TEST_VER-pytest.tar.gz" tmp_info=${TMP_INFO_FILE:-'/tmp/artifacts-build.info'} diff --git a/jenkins/version-def.sh b/jenkins/version-def.sh index dbad6d6fd94..8d363fc9d4a 100755 --- a/jenkins/version-def.sh +++ b/jenkins/version-def.sh @@ -32,7 +32,7 @@ CUDA_CLASSIFIER=${CUDA_CLASSIFIER:-"cuda11"} CLASSIFIER=${CLASSIFIER:-"$CUDA_CLASSIFIER"} # default as CUDA_CLASSIFIER for compatibility PROJECT_VER=${PROJECT_VER:-"24.08.0-SNAPSHOT"} PROJECT_TEST_VER=${PROJECT_TEST_VER:-"24.08.0-SNAPSHOT"} -SPARK_VER=${SPARK_VER:-"3.1.1"} +SPARK_VER=${SPARK_VER:-"3.2.0"} SPARK_VER_213=${SPARK_VER_213:-"3.3.0"} # Make a best attempt to set the default value for the shuffle shim. # Note that SPARK_VER for non-Apache Spark flavors (i.e. databricks, @@ -85,7 +85,7 @@ fi # PHASE_TYPE: CICD phase at which the script is called, to specify Spark shim versions. # regular: noSnapshots + snapshots # pre-release: noSnapshots only -# *: shim versions to build, e.g., PHASE_TYPE="311 321" +# *: shim versions to build, e.g., PHASE_TYPE="320 321" PHASE_TYPE=${PHASE_TYPE:-"regular"} case $PHASE_TYPE in # SPARK_SHIM_VERSIONS will be used for nightly artifact build diff --git a/pom.xml b/pom.xml index 5fb949ca34d..80cdf968a7d 100644 --- a/pom.xml +++ b/pom.xml @@ -91,68 +91,11 @@ - release311 + release320 true - - buildver - 311 - - - - 311 - ${spark311.version} - ${spark311.version} - 1.10.1 - - - delta-lake/delta-stub - api_validation - - - - release312 - - - buildver - 312 - - - - 312 - ${spark312.version} - ${spark312.version} - 1.10.1 - - - delta-lake/delta-stub - api_validation - - - - release313 - - - buildver - 313 - - - - 313 - ${spark313.version} - ${spark313.version} - 1.10.1 - - - delta-lake/delta-stub - api_validation - - - - release320 - buildver 320 @@ -709,11 +652,11 @@ . ${spark.rapids.project.basedir}/target/${spark.version.classifier}/.sbt/1.0/zinc/org.scala-sbt false - 311 + 320 1.8 1.8 8 - ${spark311.version} + ${spark320.version} ${spark.version} 1.10.1 spark${buildver} @@ -758,9 +701,6 @@ - 3.1.1 - 3.1.2 - 3.1.3 3.2.0 3.2.1 3.2.1.3.2.7171000.0-3 @@ -812,9 +752,6 @@ ${project.basedir}/target/${spark.version.classifier}/generated/src - 311, - 312, - 313, 320, 321, 321cdh, @@ -843,8 +780,8 @@ 341db @@ -863,7 +800,6 @@ 340 - 312, 321, 331, 340 diff --git a/scala2.13/aggregator/pom.xml b/scala2.13/aggregator/pom.xml index 198b62d5fa6..bf643394a98 100644 --- a/scala2.13/aggregator/pom.xml +++ b/scala2.13/aggregator/pom.xml @@ -252,79 +252,11 @@ - release311 + release320 - - buildver - 311 - - - - - com.nvidia - rapids-4-spark-delta-stub_${scala.binary.version} - ${project.version} - ${spark.version.classifier} - - - - - release312 - - - buildver - 312 - - - - - com.nvidia - rapids-4-spark-delta-stub_${scala.binary.version} - ${project.version} - ${spark.version.classifier} - - - - - release313 - - - buildver - 313 - - - - - com.nvidia - rapids-4-spark-delta-stub_${scala.binary.version} - ${project.version} - ${spark.version.classifier} - - - - - release314 - - - buildver - 314 - - - - - com.nvidia - rapids-4-spark-delta-stub_${scala.binary.version} - ${project.version} - ${spark.version.classifier} - - - - - release320 - buildver 320 diff --git a/scala2.13/dist/pom.xml b/scala2.13/dist/pom.xml index db9ada51a28..1e651257f06 100644 --- a/scala2.13/dist/pom.xml +++ b/scala2.13/dist/pom.xml @@ -145,7 +145,6 @@ minimumFeatureVersionMix - 312, 320, 321cdh, 330, @@ -389,7 +388,7 @@ self.log("... OK") - + diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index 3b6080eb56c..ad353de3602 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -91,68 +91,11 @@ - release311 + release320 - - buildver - 311 - - - - 311 - ${spark311.version} - ${spark311.version} - 1.10.1 - - - delta-lake/delta-stub - api_validation - - - - release312 - - - buildver - 312 - - - - 312 - ${spark312.version} - ${spark312.version} - 1.10.1 - - - delta-lake/delta-stub - api_validation - - - - release313 - - - buildver - 313 - - - - 313 - ${spark313.version} - ${spark313.version} - 1.10.1 - - - delta-lake/delta-stub - api_validation - - - - release320 - buildver 320 @@ -709,7 +652,7 @@ . ${spark.rapids.project.basedir}/target/${spark.version.classifier}/.sbt/1.0/zinc/org.scala-sbt false - 311 + 320 1.8 1.8 8 @@ -758,9 +701,6 @@ - 3.1.1 - 3.1.2 - 3.1.3 3.2.0 3.2.1 3.2.1.3.2.7171000.0-3 @@ -812,9 +752,6 @@ ${project.basedir}/target/${spark.version.classifier}/generated/src - 311, - 312, - 313, 320, 321, 321cdh, @@ -843,8 +780,8 @@ 341db @@ -863,7 +800,6 @@ 340 - 312, 321, 331, 340 diff --git a/sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala b/sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala index 2d7a51c4e43..59434084e1a 100644 --- a/sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala +++ b/sql-plugin-api/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala @@ -44,16 +44,16 @@ import org.apache.spark.util.MutableURLClassLoader 3. a smaller fraction of classes that differ under one of the supported Spark versions com/nvidia/spark/SQLPlugin.class spark-shared/com/nvidia/spark/rapids/CastExprMeta.class - spark311/org/apache/spark/sql/rapids/GpuUnaryMinus.class - spark320/org/apache/spark/sql/rapids/GpuUnaryMinus.class + spark320/org/apache/spark/sql/rapids/aggregate/GpuLast.class + spark331/org/apache/spark/sql/rapids/aggregate/GpuLast.class Each shim can see a consistent parallel world without conflicts by referencing only one conflicting directory. E.g., Spark 3.2.0 Shim will use jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark-shared/ jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark320/ - Spark 3.1.1 will use + Spark 3.3.1 will use jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark-shared/ - jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark311/ + jar:file:/home/spark/rapids-4-spark_2.12-24.08.0.jar!/spark331/ Using these Jar URL's allows referencing different bytecode produced from identical sources by incompatible Scala / Spark dependencies. */ diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index 8abf7e616c2..4b95ab4b6a6 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -2021,9 +2021,9 @@ val SHUFFLE_COMPRESSION_LZ4_CHUNK_SIZE = conf("spark.rapids.shuffle.compression. .startupOnly() .doc("Overrides the automatic Spark shim detection logic and forces a specific shims " + "provider class to be used. Set to the fully qualified shims provider class to use. " + - "If you are using a custom Spark version such as Spark 3.1.1.0 then this can be used to " + - "specify the shims provider that matches the base Spark version of Spark 3.1.1, i.e.: " + - "com.nvidia.spark.rapids.shims.spark311.SparkShimServiceProvider. If you modified Spark " + + "If you are using a custom Spark version such as Spark 3.2.0 then this can be used to " + + "specify the shims provider that matches the base Spark version of Spark 3.2.0, i.e.: " + + "com.nvidia.spark.rapids.shims.spark320.SparkShimServiceProvider. If you modified Spark " + "then there is no guarantee the RAPIDS Accelerator will function properly." + "When tested in a combined jar with other Shims, it's expected that the provided " + "implementation follows the same convention as existing Spark shims. If its class" + diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/ShimmedExecutionPlanCaptureCallbackImpl.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/ShimmedExecutionPlanCaptureCallbackImpl.scala index 00379026f05..b843fe1279a 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/ShimmedExecutionPlanCaptureCallbackImpl.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/ShimmedExecutionPlanCaptureCallbackImpl.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ import org.apache.spark.sql.execution.exchange.ReusedExchangeExec /** * Note that the name is prefixed with "Shimmed" such that wildcard rules - * under unshimmed-common-from-spark311.txt don't get confused and pick this class to be + * under unshimmed-common-from-spark320.txt don't get confused and pick this class to be * un-shimmed. */ class ShimmedExecutionPlanCaptureCallbackImpl extends ExecutionPlanCaptureCallbackBase { diff --git a/sql-plugin/src/main/spark311/java/com/nvidia/spark/rapids/shims/ShimSupportsRuntimeFiltering.java b/sql-plugin/src/main/spark311/java/com/nvidia/spark/rapids/shims/ShimSupportsRuntimeFiltering.java deleted file mode 100644 index 03f2f369f03..00000000000 --- a/sql-plugin/src/main/spark311/java/com/nvidia/spark/rapids/shims/ShimSupportsRuntimeFiltering.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims; - -import org.apache.spark.sql.connector.expressions.NamedReference; -import org.apache.spark.sql.sources.Filter; - -/** - * Shim interface for Apache Spark's SupportsRuntimeFiltering interface - * which was added in Spark 3.2.0. - */ -public interface ShimSupportsRuntimeFiltering { - NamedReference[] filterAttributes(); - - void filter(Filter[] filters); -} diff --git a/sql-plugin/src/main/spark311/java/com/nvidia/spark/rapids/shims/XxHash64Shims.scala b/sql-plugin/src/main/spark311/java/com/nvidia/spark/rapids/shims/XxHash64Shims.scala deleted file mode 100644 index 15cc6e1bfcb..00000000000 --- a/sql-plugin/src/main/spark311/java/com/nvidia/spark/rapids/shims/XxHash64Shims.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import com.nvidia.spark.rapids.TypeSig - -object XxHash64Shims { - // Spark 3.1.x does not normalize -0.0 to 0.0 but spark-rapids-jni kernel does - val supportedTypes: TypeSig = (TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128 - - TypeSig.FLOAT - TypeSig.DOUBLE) -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AvoidAdaptiveTransitionToRow.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AvoidAdaptiveTransitionToRow.scala deleted file mode 100644 index 166c52364af..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AvoidAdaptiveTransitionToRow.scala +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import java.lang.reflect.Method - -import com.nvidia.spark.rapids.{GpuColumnarToRowExec, GpuExec, GpuRowToColumnarExec} - -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.execution.SparkPlan -import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec -import org.apache.spark.sql.vectorized.ColumnarBatch - -/** - * This operator will attempt to optimize the case when we are writing the results of - * an adaptive query to disk so that we remove the redundant transitions from columnar - * to row within AdaptiveSparkPlanExec followed by a row to columnar transition. - * - * Specifically, this is the plan we see in this case: - * - * {{{ - * GpuRowToColumnar(AdaptiveSparkPlanExec(GpuColumnarToRow(child)) - * }}} - * - * We perform this optimization at runtime rather than during planning, because when the adaptive - * plan is being planned and executed, we don't know whether it is being called from an operation - * that wants rows (such as CollectTailExec) or from an operation that wants columns (such as - * GpuDataWritingCommandExec). - * - * Spark does not provide a mechanism for executing an adaptive plan and retrieving columnar - * results and the internal methods that we need to call are private, so we use reflection to - * call them. - * - * @param child The plan to execute - */ -case class AvoidAdaptiveTransitionToRow(child: SparkPlan) extends ShimUnaryExecNode with GpuExec { - - override def doExecute(): RDD[InternalRow] = - throw new IllegalStateException(s"Row-based execution should not occur for $this") - - override def output: Seq[Attribute] = child.output - - override protected def internalDoExecuteColumnar(): RDD[ColumnarBatch] = child match { - case GpuRowToColumnarExec(a: AdaptiveSparkPlanExec, _) => - val getFinalPhysicalPlan = getPrivateMethod("getFinalPhysicalPlan") - val plan = getFinalPhysicalPlan.invoke(a) - val rdd = plan match { - case t: GpuColumnarToRowExec => - t.child.executeColumnar() - case _ => - child.executeColumnar() - } - - // final UI update - val finalPlanUpdate = getPrivateMethod("finalPlanUpdate") - finalPlanUpdate.invoke(a) - - rdd - - case _ => - child.executeColumnar() - } - - private def getPrivateMethod(name: String): Method = { - val m = classOf[AdaptiveSparkPlanExec].getDeclaredMethod(name) - m.setAccessible(true) - m - } -} \ No newline at end of file diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala deleted file mode 100644 index 7353544ec8a..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import com.nvidia.spark.rapids._ - -import org.apache.spark.sql.execution.datasources.v2.BatchScanExec - -class BatchScanExecMeta(p: BatchScanExec, - conf: RapidsConf, - parent: Option[RapidsMeta[_, _, _]], - rule: DataFromReplacementRule) - extends SparkPlanMeta[BatchScanExec](p, conf, parent, rule) { - - override val childScans: scala.Seq[ScanMeta[_]] = - Seq(GpuOverrides.wrapScan(p.scan, conf, Some(this))) - - override def convertToGpu(): GpuExec = - GpuBatchScanExec(p.output, childScans.head.convertToGpu()) -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala deleted file mode 100644 index e7d5d452abb..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuBatchScanExec.scala +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import com.nvidia.spark.rapids.{GpuBatchScanExecMetrics, GpuScan} - -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.AttributeReference -import org.apache.spark.sql.catalyst.plans.QueryPlan -import org.apache.spark.sql.connector.read._ -import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase -import org.apache.spark.sql.vectorized.ColumnarBatch - -case class GpuBatchScanExec( - output: Seq[AttributeReference], - @transient scan: GpuScan) extends DataSourceV2ScanExecBase with GpuBatchScanExecMetrics { - @transient lazy val batch: Batch = scan.toBatch - - @transient override lazy val partitions: Seq[InputPartition] = batch.planInputPartitions() - - override lazy val readerFactory: PartitionReaderFactory = batch.createReaderFactory() - - override lazy val inputRDD: RDD[InternalRow] = { - scan.metrics = allMetrics - new GpuDataSourceRDD(sparkContext, partitions, readerFactory) - } - - override def doCanonicalize(): GpuBatchScanExec = { - this.copy(output = output.map(QueryPlan.normalizeExpressions(_, output))) - } - - override def internalDoExecuteColumnar(): RDD[ColumnarBatch] = { - val numOutputRows = longMetric("numOutputRows") - inputRDD.asInstanceOf[RDD[ColumnarBatch]].map { b => - numOutputRows += b.numRows() - b - } - } -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuDataSourceRDD.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuDataSourceRDD.scala deleted file mode 100644 index 044bbffa538..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuDataSourceRDD.scala +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import com.nvidia.spark.rapids.{MetricsBatchIterator, PartitionIterator} -import com.nvidia.spark.rapids.ScalableTaskCompletion.onTaskCompletion - -import org.apache.spark.{InterruptibleIterator, Partition, SparkContext, SparkException, TaskContext} -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.connector.read.{InputPartition, PartitionReaderFactory} -import org.apache.spark.sql.execution.datasources.v2.{DataSourceRDD, DataSourceRDDPartition} -import org.apache.spark.sql.vectorized.ColumnarBatch - -/** - * A replacement for DataSourceRDD that does NOT compute the bytes read input metric. - * DataSourceRDD assumes all reads occur on the task thread, and some GPU input sources - * use multithreaded readers that cannot generate proper metrics with DataSourceRDD. - * @note It is the responsibility of users of this RDD to generate the bytes read input - * metric explicitly! - */ -class GpuDataSourceRDD( - sc: SparkContext, - @transient private val inputPartitions: Seq[InputPartition], - partitionReaderFactory: PartitionReaderFactory) - extends DataSourceRDD(sc, inputPartitions, partitionReaderFactory, columnarReads = true) { - - private def castPartition(split: Partition): DataSourceRDDPartition = split match { - case p: DataSourceRDDPartition => p - case _ => throw new SparkException(s"[BUG] Not a DataSourceRDDPartition: $split") - } - - override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = { - val inputPartition = castPartition(split).inputPartition - val batchReader = partitionReaderFactory.createColumnarReader(inputPartition) - val iter = new MetricsBatchIterator(new PartitionIterator[ColumnarBatch](batchReader)) - onTaskCompletion(batchReader.close()) - // TODO: SPARK-25083 remove the type erasure hack in data source scan - new InterruptibleIterator(context, iter.asInstanceOf[Iterator[InternalRow]]) - } -} - -object GpuDataSourceRDD { - def apply( - sc: SparkContext, - inputPartitions: Seq[InputPartition], - partitionReaderFactory: PartitionReaderFactory): GpuDataSourceRDD = { - new GpuDataSourceRDD(sc, inputPartitions, partitionReaderFactory) - } -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala deleted file mode 100644 index 37d63f3d7b3..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOrcDataReader.scala +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import java.nio.ByteBuffer - -import com.nvidia.spark.rapids.{GpuMetric, HostMemoryOutputStream, NoopMetric} -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.hive.common.io.DiskRangeList -import org.apache.orc.{CompressionCodec, DataReader, OrcFile, OrcProto, StripeInformation, TypeDescription} -import org.apache.orc.impl.{BufferChunk, DataReaderProperties, OrcIndex, RecordReaderUtils} - -/** - * File cache is not supported for Spark 3.1.x so this is a thin wrapper - * around the ORC DataReader. - */ -class GpuOrcDataReader( - val props: DataReaderProperties, - val conf: Configuration, - val metrics: Map[String, GpuMetric]) extends DataReader { - private var dataReader = RecordReaderUtils.createDefaultDataReader(props) - - override def open(): Unit = dataReader.open() - - override def readRowIndex( - stripe: StripeInformation, - fileSchema: TypeDescription, - footer: OrcProto.StripeFooter, - ignoreNonUtf8BloomFilter: Boolean, - included: Array[Boolean], - indexes: Array[OrcProto.RowIndex], - sargColumns: Array[Boolean], - version: OrcFile.WriterVersion, - bloomFilterKinds: Array[OrcProto.Stream.Kind], - bloomFilterIndices: Array[OrcProto.BloomFilterIndex]): OrcIndex = { - dataReader.readRowIndex(stripe, fileSchema, footer, ignoreNonUtf8BloomFilter, included, - indexes, sargColumns, version, bloomFilterKinds, bloomFilterIndices) - } - - override def readStripeFooter(stripe: StripeInformation): OrcProto.StripeFooter = { - dataReader.readStripeFooter(stripe) - } - - override def readFileData( - range: DiskRangeList, - baseOffset: Long, - forceDirect: Boolean): DiskRangeList = { - dataReader.readFileData(range, baseOffset, forceDirect) - } - - override def isTrackingDiskRanges: Boolean = dataReader.isTrackingDiskRanges - - override def releaseBuffer(buffer: ByteBuffer): Unit = dataReader.releaseBuffer(buffer) - - override def getCompressionCodec: CompressionCodec = dataReader.getCompressionCodec - - override def close(): Unit = { - if (dataReader != null) { - dataReader.close() - dataReader = null - } - } - - def copyFileDataToHostStream(out: HostMemoryOutputStream, ranges: DiskRangeList): Unit = { - val bufferChunks = dataReader.readFileData(ranges, 0, false) - metrics.getOrElse(GpuMetric.WRITE_BUFFER_TIME, NoopMetric).ns { - var current = bufferChunks - while (current != null) { - out.write(current.getData) - if (dataReader.isTrackingDiskRanges && current.isInstanceOf[BufferChunk]) { - dataReader.releaseBuffer(current.getData) - } - current = current.next - } - } - } -} - -object GpuOrcDataReader { - // File cache is not being used, so we want to coalesce read ranges - val shouldMergeDiskRanges: Boolean = true -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuParquetCrypto.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuParquetCrypto.scala deleted file mode 100644 index 54dc9ef6a47..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuParquetCrypto.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -object GpuParquetCrypto { - /** - * Columnar encryption was added in Spark 3.2.0 - */ - def isColumnarCryptoException(e: Throwable): Boolean = false -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/HashUtils.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/HashUtils.scala deleted file mode 100644 index 5e381dc3ae3..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/HashUtils.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import ai.rapids.cudf - -object HashUtils { - /** - * In Spark 3.2.0+ -0.0 is normalized to 0.0, but for everyone else this is a noop - * @param in the input to normalize - * @return the result - */ - def normalizeInput(in: cudf.ColumnVector): cudf.ColumnVector = - in.incRefCount() -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala deleted file mode 100644 index 5fd65188e60..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import com.nvidia.spark.rapids.{BaseExprMeta, DataFromReplacementRule, ExprMeta, GpuOverrides, RapidsConf, RapidsMeta} - -import org.apache.spark.sql.catalyst.expressions.{Expression, Lag, Lead, Literal, OffsetWindowFunction} -import org.apache.spark.sql.types.IntegerType - -/** - * Spark 3.1.1-specific replacement for com.nvidia.spark.rapids.OffsetWindowFunctionMeta. - * This is required primarily for two reasons: - * 1. com.nvidia.spark.rapids.OffsetWindowFunctionMeta (compiled against Spark 3.0.x) - * fails class load in Spark 3.1.x. (`expr.input` is not recognized as an Expression.) - * 2. The semantics of offsets in LAG() are reversed/negated in Spark 3.1.1. - * E.g. The expression `LAG(col, 5)` causes Lag.offset to be set to `-5`, - * as opposed to `5`, in prior versions of Spark. - * This class adjusts the LAG offset to use similar semantics to Spark 3.0.x. - */ -abstract class OffsetWindowFunctionMeta[INPUT <: OffsetWindowFunction] ( - expr: INPUT, - conf: RapidsConf, - parent: Option[RapidsMeta[_, _, _]], - rule: DataFromReplacementRule) - extends ExprMeta[INPUT](expr, conf, parent, rule) { - lazy val input: BaseExprMeta[_] = GpuOverrides.wrapExpr(expr.input, conf, Some(this)) - lazy val adjustedOffset: Expression = { - expr match { - case lag: Lag => - GpuOverrides.extractLit(lag.offset) match { - case Some(Literal(offset: Int, IntegerType)) => - Literal(-offset, IntegerType) - case _ => - throw new IllegalStateException( - s"Only integer literal offsets are supported for LAG. Found:${lag.offset}") - } - case lead: Lead => - GpuOverrides.extractLit(lead.offset) match { - case Some(Literal(offset: Int, IntegerType)) => - Literal(offset, IntegerType) - case _ => - throw new IllegalStateException( - s"Only integer literal offsets are supported for LEAD. Found:${lead.offset}") - } - case other => - throw new IllegalStateException(s"$other is not a supported window function") - } - } - lazy val offset: BaseExprMeta[_] = - GpuOverrides.wrapExpr(adjustedOffset, conf, Some(this)) - lazy val default: BaseExprMeta[_] = GpuOverrides.wrapExpr(expr.default, conf, Some(this)) - - override val childExprs: Seq[BaseExprMeta[_]] = Seq(input, offset, default) - - override def tagExprForGpu(): Unit = { - expr match { - case Lead(_,_,_) => // Supported. - case Lag(_,_,_) => // Supported. - case other => - willNotWorkOnGpu( s"Only LEAD/LAG offset window functions are supported. Found: $other") - } - - if (GpuOverrides.extractLit(expr.offset).isEmpty) { // Not a literal offset. - willNotWorkOnGpu( - s"Only integer literal offsets are supported for LEAD/LAG. Found: ${expr.offset}") - } - } -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcCastingShims.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcCastingShims.scala deleted file mode 100644 index 2bbc860ca52..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcCastingShims.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import ai.rapids.cudf.{ColumnView, DType} -import com.nvidia.spark.rapids.GpuOrcScan - - -object OrcCastingShims { - - def castIntegerToTimestamp(col: ColumnView, colType: DType): ColumnView = { - // For 311 <= spark < 320 (including 311, 312, 313), they consider the integer as - // milliseconds. - GpuOrcScan.castIntegersToTimestamp(col, colType, DType.TIMESTAMP_MILLISECONDS) - } -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcShims.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcShims.scala deleted file mode 100644 index 3ded1c76620..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcShims.scala +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import com.nvidia.spark.rapids.RapidsPluginImplicits._ -import org.apache.orc.Reader - -object OrcShims extends OrcShims311until320Base { - - // the ORC Reader in non CDH Spark is closeable - def withReader[T <: AutoCloseable, V](r: T)(block: T => V): V = { - try { - block(r) - } finally { - r.safeClose() - } - } - - // the ORC Reader in non CDH Spark is closeable - def closeReader(reader: Reader): Unit = { - if (reader != null) { - reader.close() - } - } -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcShims311until320Base.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcShims311until320Base.scala deleted file mode 100644 index ec60ea61e4a..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcShims311until320Base.scala +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import java.nio.ByteBuffer - -import scala.collection.mutable.ArrayBuffer - -import com.nvidia.spark.rapids.OrcOutputStripe -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.hive.common.io.DiskRange -import org.apache.orc.{CompressionCodec, CompressionKind, DataReader, OrcFile, OrcProto, PhysicalWriter, Reader, StripeInformation, TypeDescription} -import org.apache.orc.impl.{BufferChunk, DataReaderProperties, InStream, OrcCodecPool, OutStream, SchemaEvolution} -import org.apache.orc.impl.RecordReaderImpl.SargApplier - -import org.apache.spark.sql.execution.datasources.orc.OrcUtils -import org.apache.spark.sql.types.DataType - -trait OrcShims311until320Base { - - // create reader properties builder - def newDataReaderPropertiesBuilder(compressionSize: Int, - compressionKind: CompressionKind, typeCount: Int): DataReaderProperties.Builder = { - DataReaderProperties.builder() - .withBufferSize(compressionSize) - .withCompression(compressionKind) - .withTypeCount(typeCount) - } - - // create ORC out stream - def newOrcOutStream(name: String, bufferSize: Int, codec: CompressionCodec, - receiver: PhysicalWriter.OutputReceiver): OutStream = { - new OutStream(name, bufferSize, codec, receiver) - } - - // filter stripes by pushing down filter - def filterStripes( - stripes: Seq[StripeInformation], - conf: Configuration, - orcReader: Reader, - dataReader: DataReader, - gen: (StripeInformation, OrcProto.StripeFooter, Array[Int]) => OrcOutputStripe, - evolution: SchemaEvolution, - sargApp: SargApplier, - sargColumns: Array[Boolean], - ignoreNonUtf8BloomFilter: Boolean, - writerVersion: OrcFile.WriterVersion, - fileIncluded: Array[Boolean], - columnMapping: Array[Int]): ArrayBuffer[OrcOutputStripe] = { - val result = new ArrayBuffer[OrcOutputStripe](stripes.length) - stripes.foreach { stripe => - val stripeFooter = dataReader.readStripeFooter(stripe) - val needStripe = if (sargApp != null) { - // An ORC schema is a single struct type describing the schema fields - val orcFileSchema = evolution.getFileType(0) - val orcIndex = dataReader.readRowIndex(stripe, orcFileSchema, stripeFooter, - ignoreNonUtf8BloomFilter, fileIncluded, null, sargColumns, - writerVersion, null, null) - val rowGroups = sargApp.pickRowGroups(stripe, orcIndex.getRowGroupIndex, - orcIndex.getBloomFilterKinds, stripeFooter.getColumnsList, orcIndex.getBloomFilterIndex, - true) - rowGroups != SargApplier.READ_NO_RGS - } else { - true - } - - if (needStripe) { - result.append(gen(stripe, stripeFooter, columnMapping)) - } - } - result - } - - /** - * Compare if the two TypeDescriptions are equal by ignoring attribute - */ - def typeDescriptionEqual(lhs: TypeDescription, rhs: TypeDescription): Boolean = { - lhs.equals(rhs) - } - - // forcePositionalEvolution is available from Spark-3.2. So setting this as false. - def forcePositionalEvolution(conf:Configuration): Boolean = { - false - } - - // orcTypeDescriptionString is renamed to getOrcSchemaString from 3.3+ - def getOrcSchemaString(dt: DataType): String = { - OrcUtils.orcTypeDescriptionString(dt) - } - - def parseFooterFromBuffer( - bb: ByteBuffer, - ps: OrcProto.PostScript, - psLen: Int): OrcProto.Footer = { - val footerSize = ps.getFooterLength.toInt - val footerOffset = bb.limit() - 1 - psLen - footerSize - val footerBuffer = bb.duplicate() - footerBuffer.position(footerOffset) - footerBuffer.limit(footerOffset + footerSize) - val diskRanges = new java.util.ArrayList[DiskRange]() - diskRanges.add(new BufferChunk(footerBuffer, 0)) - val compressionKind = CompressionKind.valueOf(ps.getCompression.name()) - val codec = OrcCodecPool.getCodec(compressionKind) - try { - val in = InStream.createCodedInputStream("footer", diskRanges, footerSize, codec, - ps.getCompressionBlockSize.toInt) - OrcProto.Footer.parseFrom(in) - } finally { - OrcCodecPool.returnCodec(compressionKind, codec) - } - } - - // ORC version 1.5.x doesn't have separate stripe statistics length - def getStripeStatisticsLength(ps: OrcProto.PostScript): Long = 0L -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala deleted file mode 100644 index 4d6d4967a80..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetSchemaClipShims.scala +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import org.apache.parquet.schema._ -import org.apache.parquet.schema.OriginalType._ -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._ - -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.rapids.execution.RapidsAnalysisException -import org.apache.spark.sql.rapids.shims.RapidsErrorUtils -import org.apache.spark.sql.types._ - -object ParquetSchemaClipShims { - /** Stubs for configs not defined before Spark 330 */ - def useFieldId(conf: SQLConf): Boolean = false - - def ignoreMissingIds(conf: SQLConf): Boolean = false - - def checkIgnoreMissingIds(ignoreMissingIds: Boolean, parquetFileSchema: MessageType, - catalystRequestedSchema: StructType): Unit = {} - - def hasFieldId(field: StructField): Boolean = - throw new IllegalStateException("This shim should not invoke `hasFieldId`") - - def hasFieldIds(schema: StructType): Boolean = - throw new IllegalStateException("This shim should not invoke `hasFieldIds`") - - def getFieldId(field: StructField): Int = - throw new IllegalStateException("This shim should not invoke `getFieldId`") - - def fieldIdToFieldMap(useFieldId: Boolean, fileType: Type): Map[Int, Type] = Map.empty[Int, Type] - - def fieldIdToNameMap(useFieldId: Boolean, - fileType: Type): Map[Int, String] = Map.empty[Int, String] - - /** - * Convert a Parquet primitive type to a Spark type. - * Based on Spark's ParquetSchemaConverter.convertPrimitiveField - */ - def convertPrimitiveField(field: PrimitiveType): DataType = { - val typeName = field.getPrimitiveTypeName - val originalType = field.getOriginalType - - def typeString = - if (originalType == null) s"$typeName" else s"$typeName ($originalType)" - - def typeNotSupported() = - throw new RapidsAnalysisException(s"Parquet type not supported: $typeString") - - def typeNotImplemented() = - throw RapidsErrorUtils.parquetTypeUnsupportedYetError(typeString) - - def illegalType() = - throw RapidsErrorUtils.illegalParquetTypeError(typeString) - - // When maxPrecision = -1, we skip precision range check, and always respect the precision - // specified in field.getDecimalMetadata. This is useful when interpreting decimal types stored - // as binaries with variable lengths. - def makeDecimalType(maxPrecision: Int = -1): DecimalType = { - val precision = field.getDecimalMetadata.getPrecision - val scale = field.getDecimalMetadata.getScale - - if (!(maxPrecision == -1 || 1 <= precision && precision <= maxPrecision)) { - throw new RapidsAnalysisException(s"Invalid decimal precision: $typeName " + - s"cannot store $precision digits (max $maxPrecision)") - } - - DecimalType(precision, scale) - } - - typeName match { - case BOOLEAN => BooleanType - - case FLOAT => FloatType - - case DOUBLE => DoubleType - - case INT32 => - originalType match { - case INT_8 => ByteType - case INT_16 => ShortType - case INT_32 | null => IntegerType - case DATE => DateType - case DECIMAL => makeDecimalType(Decimal.MAX_INT_DIGITS) - case UINT_8 => typeNotSupported() - case UINT_16 => typeNotSupported() - case UINT_32 => typeNotSupported() - case TIME_MILLIS => typeNotImplemented() - case _ => illegalType() - } - - case INT64 => - originalType match { - case INT_64 | null => LongType - case DECIMAL => makeDecimalType(Decimal.MAX_LONG_DIGITS) - case UINT_64 => typeNotSupported() - case TIMESTAMP_MICROS => TimestampType - case TIMESTAMP_MILLIS => TimestampType - case _ => illegalType() - } - - case INT96 => - if (!SQLConf.get.isParquetINT96AsTimestamp) { - throw new RapidsAnalysisException( - "INT96 is not supported unless it's interpreted as timestamp. " + - s"Please try to set ${SQLConf.PARQUET_INT96_AS_TIMESTAMP.key} to true.") - } - TimestampType - - case BINARY => - originalType match { - case UTF8 | ENUM | JSON => StringType - case null if SQLConf.get.isParquetBinaryAsString => StringType - case null => BinaryType - case BSON => BinaryType - case DECIMAL => makeDecimalType() - case _ => illegalType() - } - - case FIXED_LEN_BYTE_ARRAY => - originalType match { - case DECIMAL => makeDecimalType(Decimal.maxPrecisionForBytes(field.getTypeLength)) - case INTERVAL => typeNotImplemented() - case _ => illegalType() - } - - case _ => illegalType() - } - } -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/PlanShimsImpl.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/PlanShimsImpl.scala deleted file mode 100644 index 438038fb749..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/PlanShimsImpl.scala +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import com.nvidia.spark.rapids.{GpuAlias, PlanShims} - -import org.apache.spark.sql.catalyst.expressions.{Alias, Expression} -import org.apache.spark.sql.execution.SparkPlan -import org.apache.spark.sql.types.DataType - -class PlanShimsImpl extends PlanShims { - def extractExecutedPlan(plan: SparkPlan): SparkPlan = plan - def isAnsiCast(e: Expression): Boolean = AnsiCastShim.isAnsiCast(e) - - def isAnsiCastOptionallyAliased(e: Expression): Boolean = e match { - case Alias(e, _) => isAnsiCast(e) - case GpuAlias(e, _) => isAnsiCast(e) - case e => isAnsiCast(e) - } - - def extractAnsiCastTypes(e: Expression): (DataType, DataType) = { - AnsiCastShim.extractAnsiCastTypes(e) - } -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RapidsOrcScanMeta.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RapidsOrcScanMeta.scala deleted file mode 100644 index 34bd8d7cc4f..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RapidsOrcScanMeta.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import com.nvidia.spark.rapids.{DataFromReplacementRule, GpuOrcScan, GpuScan, RapidsConf, RapidsMeta, ScanMeta} - -import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan - -class RapidsOrcScanMeta( - oScan: OrcScan, - conf: RapidsConf, - parent: Option[RapidsMeta[_, _, _]], - rule: DataFromReplacementRule) - extends ScanMeta[OrcScan](oScan, conf, parent, rule) { - - override def tagSelfForGpu(): Unit = { - GpuOrcScan.tagSupport(this) - } - - override def convertToGpu(): GpuScan = - GpuOrcScan(oScan.sparkSession, - oScan.hadoopConf, - oScan.fileIndex, - oScan.dataSchema, - oScan.readDataSchema, - oScan.readPartitionSchema, - oScan.options, - oScan.pushedFilters, - oScan.partitionFilters, - oScan.dataFilters, - conf) -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RapidsParquetScanMeta.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RapidsParquetScanMeta.scala deleted file mode 100644 index 42a905c9c45..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RapidsParquetScanMeta.scala +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import com.nvidia.spark.rapids.{DataFromReplacementRule, GpuParquetScan, GpuScan, RapidsConf, RapidsMeta, ScanMeta} - -import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan - -class RapidsParquetScanMeta( - pScan: ParquetScan, - conf: RapidsConf, - parent: Option[RapidsMeta[_, _, _]], - rule: DataFromReplacementRule) - extends ScanMeta[ParquetScan](pScan, conf, parent, rule) { - - override def tagSelfForGpu(): Unit = { - GpuParquetScan.tagSupport(this) - } - - override def convertToGpu(): GpuScan = { - GpuParquetScan(pScan.sparkSession, - pScan.hadoopConf, - pScan.fileIndex, - pScan.dataSchema, - pScan.readDataSchema, - pScan.readPartitionSchema, - pScan.pushedFilters, - pScan.options, - pScan.partitionFilters, - pScan.dataFilters, - conf) - } -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimAQEShuffleReadExec.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimAQEShuffleReadExec.scala deleted file mode 100644 index 8a4162a2175..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimAQEShuffleReadExec.scala +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import com.nvidia.spark.rapids._ - -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.execution.adaptive._ -import org.apache.spark.sql.rapids.execution._ - -class GpuCustomShuffleReaderMeta(reader: CustomShuffleReaderExec, - conf: RapidsConf, - parent: Option[RapidsMeta[_, _, _]], - rule: DataFromReplacementRule) - extends SparkPlanMeta[CustomShuffleReaderExec](reader, conf, parent, rule) { - - - override def checkExistingTags(): Unit = { - // CoalesceShufflePartitions performs a transformUp and may replace ShuffleQueryStageExec - // with CustomShuffleReaderExec, causing tags to be copied from ShuffleQueryStageExec to - // CustomShuffleReaderExec, including the "no need to replace ShuffleQueryStageExec" tag. - wrapped.getTagValue(RapidsMeta.gpuSupportedTag) - .foreach(_.diff(cannotBeReplacedReasons.get) - .filterNot(_ == s"there is no need to replace ${classOf[ShuffleQueryStageExec]}") - .foreach(willNotWorkOnGpu)) - } - - override def tagPlanForGpu(): Unit = { - if (!reader.child.supportsColumnar) { - willNotWorkOnGpu( - "Unable to replace CustomShuffleReader due to child not being columnar") - } - val shuffleEx = reader.child.asInstanceOf[ShuffleQueryStageExec].plan - shuffleEx.getTagValue(GpuOverrides.preRowToColProjection).foreach { r2c => - wrapped.setTagValue(GpuOverrides.preRowToColProjection, r2c) - } - } - - override def convertToGpu(): GpuExec = { - GpuCustomShuffleReaderExec(childPlans.head.convertIfNeeded(), - reader.partitionSpecs) - } - - // extract output attributes of the underlying ShuffleExchange - override def outputAttributes: Seq[Attribute] = { - val shuffleEx = reader.child.asInstanceOf[ShuffleQueryStageExec].plan - shuffleEx.getTagValue(GpuShuffleMetaBase.shuffleExOutputAttributes) - .getOrElse(shuffleEx.output) - } - - // fetch availableRuntimeDataTransition of the underlying ShuffleExchange - override val availableRuntimeDataTransition: Boolean = { - val shuffleEx = reader.child.asInstanceOf[ShuffleQueryStageExec].plan - shuffleEx.getTagValue(GpuShuffleMetaBase.availableRuntimeDataTransition) - .getOrElse(false) - } -} \ No newline at end of file diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimBaseSubqueryExec.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimBaseSubqueryExec.scala deleted file mode 100644 index 9c28a0b01df..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimBaseSubqueryExec.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.execution.BaseSubqueryExec - -trait ShimBaseSubqueryExec extends BaseSubqueryExec { - @transient final val session = SparkSession.getActiveSession.orNull -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimBroadcastExchangeLike.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimBroadcastExchangeLike.scala deleted file mode 100644 index f86d7e51ca6..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimBroadcastExchangeLike.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import scala.concurrent.Promise - -import org.apache.spark.broadcast.Broadcast -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.execution.exchange.BroadcastExchangeLike - -/** - * This shim handles the completion future differences between - * Apache Spark and Databricks. - */ -trait ShimBroadcastExchangeLike extends BroadcastExchangeLike { - @transient final val session = SparkSession.getActiveSession.orNull - - @transient - protected lazy val promise = Promise[Broadcast[Any]]() - - /** - * For registering callbacks on `relationFuture`. - * Note that calling this field will not start the execution of broadcast job. - */ - @transient - lazy val completionFuture: concurrent.Future[Broadcast[Any]] = promise.future -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimPredicateHelper.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimPredicateHelper.scala deleted file mode 100644 index 2e229fab1df..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimPredicateHelper.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.rapids._ - -trait ShimPredicateHelper extends PredicateHelper { - // SPARK-30027 from 3.2.0 - // If one expression and its children are null intolerant, it is null intolerant. - protected def isNullIntolerant(expr: Expression): Boolean = expr match { - case e: NullIntolerant => e.children.forall(isNullIntolerant) - case _ => false - } - - override protected def splitConjunctivePredicates( - condition: Expression - ): Seq[Expression] = { - condition match { - case GpuAnd(cond1, cond2) => - splitConjunctivePredicates(cond1) ++ splitConjunctivePredicates(cond2) - case _ => super.splitConjunctivePredicates(condition) - } - } -} \ No newline at end of file diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShuffleOriginUtil.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShuffleOriginUtil.scala deleted file mode 100644 index ef7d256d41f..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShuffleOriginUtil.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, REPARTITION, REPARTITION_WITH_NUM, ShuffleOrigin} - -object ShuffleOriginUtil { - private val knownOrigins: Set[ShuffleOrigin] = Set(ENSURE_REQUIREMENTS, - REPARTITION, REPARTITION_WITH_NUM) - - def isSupported(origin: ShuffleOrigin): Boolean = knownOrigins.contains(origin) -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/Spark31XShims.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/Spark31XShims.scala deleted file mode 100644 index eab56a8348a..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/Spark31XShims.scala +++ /dev/null @@ -1,441 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import scala.collection.mutable.ListBuffer - -import com.nvidia.spark.rapids._ -import org.apache.hadoop.fs.FileStatus - -import org.apache.spark.internal.Logging -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} -import org.apache.spark.sql.catalyst.errors.attachTree -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.aggregate.Average -import org.apache.spark.sql.catalyst.plans.physical.BroadcastMode -import org.apache.spark.sql.catalyst.trees.TreeNode -import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils} -import org.apache.spark.sql.connector.read.Scan -import org.apache.spark.sql.execution._ -import org.apache.spark.sql.execution.adaptive._ -import org.apache.spark.sql.execution.command.{AlterTableRecoverPartitionsCommand, RunnableCommand} -import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.datasources.v2.orc.OrcScan -import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan -import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec} -import org.apache.spark.sql.execution.joins._ -import org.apache.spark.sql.execution.python._ -import org.apache.spark.sql.execution.window.WindowExecBase -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.rapids._ -import org.apache.spark.sql.rapids.aggregate._ -import org.apache.spark.sql.rapids.execution.GpuCustomShuffleReaderExec -import org.apache.spark.sql.rapids.execution.python._ -import org.apache.spark.sql.types._ - -// 31x nondb shims, used by 311cdh and 31x -abstract class Spark31XShims extends Spark31Xuntil33XShims with Logging { - override def parquetRebaseReadKey: String = - SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key - override def parquetRebaseWriteKey: String = - SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key - override def avroRebaseReadKey: String = - SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key - override def avroRebaseWriteKey: String = - SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key - override def parquetRebaseRead(conf: SQLConf): String = - conf.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ) - override def parquetRebaseWrite(conf: SQLConf): String = - conf.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE) - - override def sessionFromPlan(plan: SparkPlan): SparkSession = { - plan.sqlContext.sparkSession - } - - override def filesFromFileIndex(fileIndex: PartitioningAwareFileIndex): Seq[FileStatus] = { - fileIndex.allFiles() - } - - def broadcastModeTransform(mode: BroadcastMode, rows: Array[InternalRow]): Any = - mode.transform(rows) - - override def newBroadcastQueryStageExec( - old: BroadcastQueryStageExec, - newPlan: SparkPlan): BroadcastQueryStageExec = BroadcastQueryStageExec(old.id, newPlan) - - override def getDateFormatter(): DateFormatter = { - DateFormatter(DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)) - } - - override def isExchangeOp(plan: SparkPlanMeta[_]): Boolean = { - // if the child query stage already executed on GPU then we need to keep the - // next operator on GPU in these cases - SQLConf.get.adaptiveExecutionEnabled && (plan.wrapped match { - case _: CustomShuffleReaderExec - | _: ShuffledHashJoinExec - | _: BroadcastHashJoinExec - | _: BroadcastExchangeExec - | _: BroadcastNestedLoopJoinExec => true - case _ => false - }) - } - - override def isAqePlan(p: SparkPlan): Boolean = p match { - case _: AdaptiveSparkPlanExec | - _: QueryStageExec | - _: CustomShuffleReaderExec => true - case _ => false - } - - override def isCustomReaderExec(x: SparkPlan): Boolean = x match { - case _: GpuCustomShuffleReaderExec | _: CustomShuffleReaderExec => true - case _ => false - } - - override def aqeShuffleReaderExec: ExecRule[_ <: SparkPlan] = - GpuOverrides.exec[CustomShuffleReaderExec]( - "A wrapper of shuffle query stage", - ExecChecks((TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128 + TypeSig.ARRAY + - TypeSig.STRUCT + TypeSig.MAP + TypeSig.BINARY).nested(), TypeSig.all), - (exec, conf, p, r) => new GpuCustomShuffleReaderMeta(exec, conf, p, r)) - - override def findOperators(plan: SparkPlan, predicate: SparkPlan => Boolean): Seq[SparkPlan] = { - def recurse( - plan: SparkPlan, - predicate: SparkPlan => Boolean, - accum: ListBuffer[SparkPlan]): Seq[SparkPlan] = { - if (predicate(plan)) { - accum += plan - } - plan match { - case a: AdaptiveSparkPlanExec => recurse(a.executedPlan, predicate, accum) - case qs: BroadcastQueryStageExec => recurse(qs.broadcast, predicate, accum) - case qs: ShuffleQueryStageExec => recurse(qs.shuffle, predicate, accum) - case other => other.children.flatMap(p => recurse(p, predicate, accum)).headOption - } - accum - } - recurse(plan, predicate, new ListBuffer[SparkPlan]()) - } - - override def skipAssertIsOnTheGpu(plan: SparkPlan): Boolean = false - - override def shouldFailDivOverflow: Boolean = false - - override def leafNodeDefaultParallelism(ss: SparkSession): Int = { - ss.sparkContext.defaultParallelism - } - - override def v1RepairTableCommand(tableName: TableIdentifier): RunnableCommand = - AlterTableRecoverPartitionsCommand(tableName) - - override def isWindowFunctionExec(plan: SparkPlan): Boolean = plan.isInstanceOf[WindowExecBase] - - override def getScans: Map[Class[_ <: Scan], ScanRule[_ <: Scan]] = Seq( - GpuOverrides.scan[ParquetScan]( - "Parquet parsing", - (a, conf, p, r) => new RapidsParquetScanMeta(a, conf, p, r)), - GpuOverrides.scan[OrcScan]( - "ORC parsing", - (a, conf, p, r) => new RapidsOrcScanMeta(a, conf, p, r)) - ).map(r => (r.getClassFor.asSubclass(classOf[Scan]), r)).toMap - override def hasAliasQuoteFix: Boolean = false - - override def reusedExchangeExecPfn: PartialFunction[SparkPlan, ReusedExchangeExec] = { - case ShuffleQueryStageExec(_, e: ReusedExchangeExec) => e - case BroadcastQueryStageExec(_, e: ReusedExchangeExec) => e - } - - override def int96ParquetRebaseRead(conf: SQLConf): String = - conf.getConf(SQLConf.LEGACY_PARQUET_INT96_REBASE_MODE_IN_READ) - override def int96ParquetRebaseWrite(conf: SQLConf): String = - conf.getConf(SQLConf.LEGACY_PARQUET_INT96_REBASE_MODE_IN_WRITE) - override def int96ParquetRebaseReadKey: String = - SQLConf.LEGACY_PARQUET_INT96_REBASE_MODE_IN_READ.key - override def int96ParquetRebaseWriteKey: String = - SQLConf.LEGACY_PARQUET_INT96_REBASE_MODE_IN_WRITE.key - - override def tryTransformIfEmptyRelation(mode: BroadcastMode): Option[Any] = { - Some(broadcastModeTransform(mode, Array.empty)).filter(isEmptyRelation) - } - - override def isEmptyRelation(relation: Any): Boolean = relation match { - case EmptyHashedRelation => true - case arr: Array[InternalRow] if arr.isEmpty => true - case _ => false - } - - override def ansiCastRule: ExprRule[_ <: Expression] = { - GpuOverrides.expr[AnsiCast]( - "Convert a column of one type of data into another type", - new CastChecks { - import TypeSig._ - // nullChecks are the same - - override val booleanChecks: TypeSig = integral + fp + BOOLEAN + STRING + DECIMAL_128 - override val sparkBooleanSig: TypeSig = cpuNumeric + BOOLEAN + STRING - - override val integralChecks: TypeSig = gpuNumeric + BOOLEAN + STRING - override val sparkIntegralSig: TypeSig = cpuNumeric + BOOLEAN + STRING - - override val fpChecks: TypeSig = (gpuNumeric + BOOLEAN + STRING) - .withPsNote(TypeEnum.STRING, fpToStringPsNote) - override val sparkFpSig: TypeSig = cpuNumeric + BOOLEAN + STRING - - override val dateChecks: TypeSig = TIMESTAMP + DATE + STRING - override val sparkDateSig: TypeSig = TIMESTAMP + DATE + STRING - - override val timestampChecks: TypeSig = TIMESTAMP + DATE + STRING - override val sparkTimestampSig: TypeSig = TIMESTAMP + DATE + STRING - - // stringChecks are the same - // binaryChecks are the same - override val decimalChecks: TypeSig = gpuNumeric + STRING - override val sparkDecimalSig: TypeSig = cpuNumeric + BOOLEAN + STRING - - // calendarChecks are the same - - override val arrayChecks: TypeSig = - ARRAY.nested(commonCudfTypes + DECIMAL_128 + NULL + ARRAY + BINARY + STRUCT) + - psNote(TypeEnum.ARRAY, "The array's child type must also support being cast to " + - "the desired child type") - override val sparkArraySig: TypeSig = ARRAY.nested(all) - - override val mapChecks: TypeSig = - MAP.nested(commonCudfTypes + DECIMAL_128 + NULL + ARRAY + BINARY + STRUCT + MAP) + - psNote(TypeEnum.MAP, "the map's key and value must also support being cast to the " + - "desired child types") - override val sparkMapSig: TypeSig = MAP.nested(all) - - override val structChecks: TypeSig = - STRUCT.nested(commonCudfTypes + DECIMAL_128 + NULL + ARRAY + BINARY + STRUCT) + - psNote(TypeEnum.STRUCT, "the struct's children must also support being cast to the " + - "desired child type(s)") - override val sparkStructSig: TypeSig = STRUCT.nested(all) - - override val udtChecks: TypeSig = none - override val sparkUdtSig: TypeSig = UDT - }, - (cast, conf, p, r) => new CastExprMeta[AnsiCast](cast, GpuEvalMode.ANSI, conf = conf, - parent = p, rule = r, doFloatToIntCheck = true, stringToAnsiDate = false)) - } - - override def getExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Seq( - GpuOverrides.expr[Cast]( - "Convert a column of one type of data into another type", - new CastChecks(), - (cast, conf, p, r) => new CastExprMeta[Cast](cast, - AnsiCastShim.getEvalMode(cast), conf, p, r, - doFloatToIntCheck = true, stringToAnsiDate = false)), - GpuOverrides.expr[Average]( - "Average aggregate operator", - ExprChecks.fullAgg( - TypeSig.DOUBLE + TypeSig.DECIMAL_128, - TypeSig.DOUBLE + TypeSig.DECIMAL_128, - Seq(ParamCheck("input", - TypeSig.integral + TypeSig.fp + TypeSig.DECIMAL_128, - TypeSig.cpuNumeric))), - (a, conf, p, r) => new AggExprMeta[Average](a, conf, p, r) { - override def tagAggForGpu(): Unit = { - GpuOverrides.checkAndTagFloatAgg(a.child.dataType, conf, this) - } - - override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = - GpuAverage(childExprs.head) - - // Average is not supported in ANSI mode right now, no matter the type - override val ansiTypeToCheck: Option[DataType] = None - }), - GpuOverrides.expr[Abs]( - "Absolute value", - ExprChecks.unaryProjectAndAstInputMatchesOutput( - TypeSig.implicitCastsAstTypes, TypeSig.gpuNumeric, - TypeSig.cpuNumeric), - (a, conf, p, r) => new UnaryAstExprMeta[Abs](a, conf, p, r) { - // ANSI support for ABS was added in 3.2.0 SPARK-33275 - override def convertToGpu(child: Expression): GpuExpression = GpuAbs(child, false) - }) - ).map(r => (r.getClassFor.asSubclass(classOf[Expression]), r)).toMap - - override def getExecs: Map[Class[_ <: SparkPlan], ExecRule[_ <: SparkPlan]] = { - Seq( - GpuOverrides.exec[WindowInPandasExec]( - "The backend for Window Aggregation Pandas UDF, Accelerates the data transfer between" + - " the Java process and the Python process. It also supports scheduling GPU resources" + - " for the Python process when enabled. For now it only supports row based window frame.", - ExecChecks( - (TypeSig.commonCudfTypes + TypeSig.ARRAY).nested(TypeSig.commonCudfTypes), - TypeSig.all), - (winPy, conf, p, r) => new GpuWindowInPandasExecMetaBase(winPy, conf, p, r) { - override val windowExpressions: Seq[BaseExprMeta[NamedExpression]] = - winPy.windowExpression.map(GpuOverrides.wrapExpr(_, conf, Some(this))) - - override def convertToGpu(): GpuExec = { - GpuWindowInPandasExec( - windowExpressions.map(_.convertToGpu()), - partitionSpec.map(_.convertToGpu()), - // leave ordering expression on the CPU, it's not used for GPU computation - winPy.orderSpec, - childPlans.head.convertIfNeeded() - )(winPy.partitionSpec) - } - }).disabledByDefault("it only supports row based frame for now"), - GpuOverrides.exec[FileSourceScanExec]( - "Reading data from files, often from Hive tables", - ExecChecks((TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.STRUCT + TypeSig.MAP + - TypeSig.ARRAY + TypeSig.BINARY + TypeSig.DECIMAL_128).nested(), TypeSig.all), - (fsse, conf, p, r) => new SparkPlanMeta[FileSourceScanExec](fsse, conf, p, r) { - - // Replaces SubqueryBroadcastExec inside dynamic pruning filters with GPU counterpart - // if possible. Instead regarding filters as childExprs of current Meta, we create - // a new meta for SubqueryBroadcastExec. The reason is that the GPU replacement of - // FileSourceScan is independent from the replacement of the partitionFilters. It is - // possible that the FileSourceScan is on the CPU, while the dynamic partitionFilters - // are on the GPU. And vice versa. - private lazy val partitionFilters = { - val convertBroadcast = (bc: SubqueryBroadcastExec) => { - val meta = GpuOverrides.wrapAndTagPlan(bc, conf) - meta.tagForExplain() - meta.convertIfNeeded().asInstanceOf[BaseSubqueryExec] - } - wrapped.partitionFilters.map { filter => - filter.transformDown { - case dpe @ DynamicPruningExpression(inSub: InSubqueryExec) => - inSub.plan match { - case bc: SubqueryBroadcastExec => - dpe.copy(inSub.copy(plan = convertBroadcast(bc))) - case reuse @ ReusedSubqueryExec(bc: SubqueryBroadcastExec) => - dpe.copy(inSub.copy(plan = reuse.copy(convertBroadcast(bc)))) - case _ => - dpe - } - } - } - } - - // partition filters and data filters are not run on the GPU - override val childExprs: Seq[ExprMeta[_]] = Seq.empty - - override def tagPlanForGpu(): Unit = GpuFileSourceScanExec.tagSupport(this) - - override def convertToCpu(): SparkPlan = { - val cpu = wrapped.copy(partitionFilters = partitionFilters) - cpu.copyTagsFrom(wrapped) - cpu - } - - override def convertToGpu(): GpuExec = { - val sparkSession = wrapped.relation.sparkSession - val options = wrapped.relation.options - val (location, alluxioPathsToReplaceMap) = - if (AlluxioCfgUtils.enabledAlluxioReplacementAlgoConvertTime(conf)) { - val shouldReadFromS3 = wrapped.relation.location match { - // Only handle InMemoryFileIndex - // - // skip handle `MetadataLogFileIndex`, from the description of this class: - // it's about the files generated by the `FileStreamSink`. - // The streaming data source is not in our scope. - // - // For CatalogFileIndex and FileIndex of `delta` data source, - // need more investigation. - case inMemory: InMemoryFileIndex => - // List all the partitions to reduce overhead, pass in 2 empty filters. - // Subsequent process will do the right partition pruning. - // This operation is fast, because it lists files from the caches and the caches - // already exist in the `InMemoryFileIndex`. - val pds = inMemory.listFiles(Seq.empty, Seq.empty) - AlluxioUtils.shouldReadDirectlyFromS3(conf, pds) - case _ => - false - } - - if (!shouldReadFromS3) { - // it's convert time algorithm and some paths are not large tables - AlluxioUtils.replacePathIfNeeded( - conf, - wrapped.relation, - partitionFilters, - wrapped.dataFilters) - } else { - // convert time algorithm and read large files - (wrapped.relation.location, None) - } - } else { - // it's not convert time algorithm or read large files, do not replace - (wrapped.relation.location, None) - } - - val newRelation = HadoopFsRelation( - location, - wrapped.relation.partitionSchema, - wrapped.relation.dataSchema, - wrapped.relation.bucketSpec, - GpuFileSourceScanExec.convertFileFormat(wrapped.relation.fileFormat), - options)(sparkSession) - - GpuFileSourceScanExec( - newRelation, - wrapped.output, - wrapped.requiredSchema, - partitionFilters, - wrapped.optionalBucketSet, - wrapped.optionalNumCoalescedBuckets, - wrapped.dataFilters, - wrapped.tableIdentifier, - wrapped.disableBucketedScan, - queryUsesInputFile = false, - alluxioPathsToReplaceMap)(conf) - } - }) - ).map(r => (r.getClassFor.asSubclass(classOf[SparkPlan]), r)).toMap - } - - /** dropped by SPARK-34234 */ - override def attachTreeIfSupported[TreeType <: TreeNode[_], A]( - tree: TreeType, - msg: String)( - f: => A - ): A = { - attachTree(tree, msg)(f) - } - - override def getAdaptiveInputPlan(adaptivePlan: AdaptiveSparkPlanExec): SparkPlan = { - adaptivePlan.inputPlan - } - - override def hasCastFloatTimestampUpcast: Boolean = false - - override def isCastingStringToNegDecimalScaleSupported: Boolean = false - - override def supportsColumnarAdaptivePlans: Boolean = false - - override def columnarAdaptivePlan(a: AdaptiveSparkPlanExec, goal: CoalesceSizeGoal): SparkPlan = { - // When the input is an adaptive plan we do not get to see the GPU version until - // the plan is executed and sometimes the plan will have a GpuColumnarToRowExec as the - // final operator and we can bypass this to keep the data columnar by inserting - // the [[AvoidAdaptiveTransitionToRow]] operator here - AvoidAdaptiveTransitionToRow(GpuRowToColumnarExec(a, goal)) - } - - override def reproduceEmptyStringBug: Boolean = false -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/SparkShims.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/SparkShims.scala deleted file mode 100644 index f5b4709b983..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/SparkShims.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import org.apache.parquet.schema.MessageType - -import org.apache.spark.sql.execution.datasources.parquet.ParquetFilters - -object SparkShimImpl extends Spark31XShims { - override def hasCastFloatTimestampUpcast: Boolean = false - - override def reproduceEmptyStringBug: Boolean = true - - override def getParquetFilters( - schema: MessageType, - pushDownDate: Boolean, - pushDownTimestamp: Boolean, - pushDownDecimal: Boolean, - pushDownStartWith: Boolean, - pushDownInFilterThreshold: Int, - caseSensitive: Boolean, - lookupFileMeta: String => String, - dateTimeRebaseModeFromConf: String): ParquetFilters = { - new ParquetFilters(schema, pushDownDate, pushDownTimestamp, pushDownDecimal, pushDownStartWith, - pushDownInFilterThreshold, caseSensitive) - } -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/TreeNode.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/TreeNode.scala deleted file mode 100644 index 671bdbd45bf..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/TreeNode.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, TernaryExpression, UnaryExpression} -import org.apache.spark.sql.catalyst.plans.logical.Command -import org.apache.spark.sql.execution.{BinaryExecNode, SparkPlan, UnaryExecNode} - -trait ShimExpression extends Expression - -trait ShimUnaryExpression extends UnaryExpression - -trait ShimBinaryExpression extends BinaryExpression - -trait ShimTernaryExpression extends TernaryExpression { - def first: Expression - def second: Expression - def third: Expression - final def children: Seq[Expression] = IndexedSeq(first, second, third) -} - -trait ShimSparkPlan extends SparkPlan - -trait ShimUnaryExecNode extends UnaryExecNode - -trait ShimBinaryExecNode extends BinaryExecNode - -trait ShimUnaryCommand extends Command diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala deleted file mode 100644 index ab4fab48a45..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import com.nvidia.spark.rapids.{TypeEnum, TypeSig, TypeSigUtilBase} - -import org.apache.spark.sql.types.DataType - -/** TypeSig Support for [3.1.1, 3.2.0) */ -object TypeSigUtil extends TypeSigUtilBase { - - /** - * Check if this type of Spark-specific is supported by the plugin or not. - * - * @param check the Supported Types - * @param dataType the data type to be checked - * @return true if it is allowed else false. - */ - override def isSupported( - check: TypeEnum.ValueSet, - dataType: DataType): Boolean = false - - /** - * Get all supported types for the spark-specific - * - * @return the all supported typ - */ - override def getAllSupportedTypes(): TypeEnum.ValueSet = - TypeEnum.values - TypeEnum.DAYTIME - TypeEnum.YEARMONTH - - /** - * Return the reason why this type is not supported.\ - * - * @param check the Supported Types - * @param dataType the data type to be checked - * @param notSupportedReason the reason for not supporting - * @return the reason - */ - override def reasonNotSupported( - check: TypeEnum.ValueSet, - dataType: DataType, - notSupportedReason: Seq[String]): Seq[String] = notSupportedReason - - /** - * Map DataType to TypeEnum - * - * @param dataType the data type to be mapped - * @return the TypeEnum - */ - override def mapDataTypeToTypeEnum(dataType: DataType): TypeEnum.Value = TypeEnum.UDT - - /** Get numeric and interval TypeSig */ - override def getNumericAndInterval(): TypeSig = - TypeSig.cpuNumeric + TypeSig.CALENDAR - - /** Get Ansi year-month and day-time TypeSig, begins from 320+ */ - override def getAnsiInterval: TypeSig = TypeSig.none -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/YearParseUtil.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/YearParseUtil.scala deleted file mode 100644 index 54f3b4b60fb..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/YearParseUtil.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import com.nvidia.spark.rapids.{RapidsConf, RapidsMeta} - -object YearParseUtil { - def tagParseStringAsDate(conf: RapidsConf, meta: RapidsMeta[_, _, _]): Unit = { - // NOOP for anything prior to 3.2.0 - } -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala deleted file mode 100644 index 4a27408789c..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import com.nvidia.spark.rapids.{DataFromReplacementRule, RapidsConf, RapidsMeta} -import com.nvidia.spark.rapids.window.{GpuSpecifiedWindowFrameMetaBase, GpuWindowExpressionMetaBase, ParsedBoundary} - -import org.apache.spark.sql.catalyst.expressions.{Expression, SpecifiedWindowFrame, WindowExpression} -import org.apache.spark.sql.types.{CalendarIntervalType, DataType, DateType, IntegerType, TimestampType} - -class GpuSpecifiedWindowFrameMeta( - windowFrame: SpecifiedWindowFrame, - conf: RapidsConf, - parent: Option[RapidsMeta[_,_,_]], - rule: DataFromReplacementRule) - extends GpuSpecifiedWindowFrameMetaBase(windowFrame, conf, parent, rule) {} - -class GpuWindowExpressionMeta( - windowExpression: WindowExpression, - conf: RapidsConf, - parent: Option[RapidsMeta[_,_,_]], - rule: DataFromReplacementRule) - extends GpuWindowExpressionMetaBase(windowExpression, conf, parent, rule) {} - -object GpuWindowUtil { - - /** - * Check if the type of RangeFrame is valid in GpuWindowSpecDefinition - * @param orderSpecType the first order by data type - * @param ft the first frame boundary data type - * @return true to valid, false to invalid - */ - def isValidRangeFrameType(orderSpecType: DataType, ft: DataType): Boolean = { - (orderSpecType, ft) match { - case (DateType, IntegerType) => true - case (TimestampType, CalendarIntervalType) => true - case (a, b) => a == b - } - } - - def getRangeBoundaryValue(boundary: Expression): ParsedBoundary = boundary match { - case anything => throw new UnsupportedOperationException("Unsupported window frame" + - s" expression $anything") - } -} - diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala deleted file mode 100644 index c764799dba8..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark311 - -import com.nvidia.spark.rapids.SparkShimVersion - -object SparkShimServiceProvider { - val VERSION = SparkShimVersion(3, 1, 1) - val VERSIONNAMES = Seq(s"$VERSION") -} - -class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { - - override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION - - def matchesVersion(version: String): Boolean = { - SparkShimServiceProvider.VERSIONNAMES.contains(version) - } -} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/spark311/RapidsShuffleManager.scala b/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/spark311/RapidsShuffleManager.scala deleted file mode 100644 index 9a65f836163..00000000000 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/spark311/RapidsShuffleManager.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.spark311 - -import org.apache.spark.SparkConf -import org.apache.spark.sql.rapids.ProxyRapidsShuffleInternalManagerBase - -/** A shuffle manager optimized for the RAPIDS Plugin for Apache Spark. */ -sealed class RapidsShuffleManager( - conf: SparkConf, - isDriver: Boolean -) extends ProxyRapidsShuffleInternalManagerBase(conf, isDriver) - diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleBlockResolver.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleBlockResolver.scala deleted file mode 100644 index e4a2451c59b..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleBlockResolver.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.rapids.shims - -import com.nvidia.spark.rapids.ShuffleBufferCatalog - -import org.apache.spark.shuffle.IndexShuffleBlockResolver -import org.apache.spark.sql.rapids.GpuShuffleBlockResolverBase - -class GpuShuffleBlockResolver(resolver: IndexShuffleBlockResolver, catalog: ShuffleBufferCatalog) - extends GpuShuffleBlockResolverBase(resolver, catalog) - diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/ShuffledBatchRDDUtil.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/ShuffledBatchRDDUtil.scala deleted file mode 100644 index 54d79eae6ff..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/ShuffledBatchRDDUtil.scala +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.rapids.shims - -import org.apache.spark.{MapOutputTrackerMaster, Partition, ShuffleDependency, SparkEnv, TaskContext} -import org.apache.spark.shuffle.ShuffleReader -import org.apache.spark.sql.execution.{CoalescedPartitionSpec, PartialMapperPartitionSpec, PartialReducerPartitionSpec} -import org.apache.spark.sql.execution.metric.SQLShuffleReadMetricsReporter -import org.apache.spark.sql.rapids.execution.ShuffledBatchRDDPartition -import org.apache.spark.sql.vectorized.ColumnarBatch -import org.apache.spark.storage.{BlockId, BlockManagerId} - -/** - * Some APIs for the ShuffledBatchRDD are only accessible from org.apache.spark... - * This code tries to match the Spark code as closely as possible. Fixing a compiler or IDE - * warning is not always the best thing here because if it changes how it matches up with the - * Spark code it may make it harder to maintain as thing change in Spark. - */ -object ShuffledBatchRDDUtil { - def preferredLocations( - partition: Partition, - dependency: ShuffleDependency[Int, ColumnarBatch, ColumnarBatch]): Seq[String] = { - val tracker = SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster] - partition.asInstanceOf[ShuffledBatchRDDPartition].spec match { - case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) => - // TODO order by partition size. - startReducerIndex.until(endReducerIndex).flatMap { reducerIndex => - tracker.getPreferredLocationsForShuffle(dependency, reducerIndex) - } - - case PartialReducerPartitionSpec(_, startMapIndex, endMapIndex, _) => - tracker.getMapLocation(dependency, startMapIndex, endMapIndex) - - case PartialMapperPartitionSpec(mapIndex, _, _) => - tracker.getMapLocation(dependency, mapIndex, mapIndex + 1) - } - } - - private def getPartitionSize( - blocksByAddress: Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])]): Long = { - blocksByAddress.flatMap { case (_, blockInfos) => - blockInfos.map { case (_, size, _) => size } - }.sum - } - - def getReaderAndPartSize( - split: Partition, - context: TaskContext, - dependency: ShuffleDependency[Int, ColumnarBatch, ColumnarBatch], - sqlMetricsReporter: SQLShuffleReadMetricsReporter): - (ShuffleReader[Nothing, Nothing], Long) = { - split.asInstanceOf[ShuffledBatchRDDPartition].spec match { - case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) => - val reader = SparkEnv.get.shuffleManager.getReader( - dependency.shuffleHandle, - startReducerIndex, - endReducerIndex, - context, - sqlMetricsReporter) - val blocksByAddress = SparkEnv.get.mapOutputTracker.getMapSizesByExecutorId( - dependency.shuffleHandle.shuffleId, - 0, - Int.MaxValue, - startReducerIndex, - endReducerIndex) - (reader, getPartitionSize(blocksByAddress)) - case PartialReducerPartitionSpec(reducerIndex, startMapIndex, endMapIndex, _) => - val reader = SparkEnv.get.shuffleManager.getReader( - dependency.shuffleHandle, - startMapIndex, - endMapIndex, - reducerIndex, - reducerIndex + 1, - context, - sqlMetricsReporter) - val blocksByAddress = SparkEnv.get.mapOutputTracker.getMapSizesByExecutorId( - dependency.shuffleHandle.shuffleId, - startMapIndex, - endMapIndex, - reducerIndex, - reducerIndex + 1) - (reader, getPartitionSize(blocksByAddress)) - case PartialMapperPartitionSpec(mapIndex, startReducerIndex, endReducerIndex) => - val reader = SparkEnv.get.shuffleManager.getReader( - dependency.shuffleHandle, - mapIndex, - mapIndex + 1, - startReducerIndex, - endReducerIndex, - context, - sqlMetricsReporter) - val blocksByAddress = SparkEnv.get.mapOutputTracker.getMapSizesByExecutorId( - dependency.shuffleHandle.shuffleId, - mapIndex, - mapIndex + 1, - startReducerIndex, - endReducerIndex) - (reader, getPartitionSize(blocksByAddress)) - } - } -} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/storage/ShimDiskBlockManager.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/storage/ShimDiskBlockManager.scala deleted file mode 100644 index 4822afd76f4..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/storage/ShimDiskBlockManager.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.rapids.shims.storage - -import org.apache.spark.SparkConf -import org.apache.spark.storage.DiskBlockManager - - -class ShimDiskBlockManager(conf: SparkConf, deleteFilesOnStop: Boolean) - extends DiskBlockManager(conf, deleteFilesOnStop) \ No newline at end of file diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/ShimVectorizedColumnReader.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/ShimVectorizedColumnReader.scala deleted file mode 100644 index 23205395210..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/ShimVectorizedColumnReader.scala +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.execution.datasources.parquet.rapids.shims - -import java.time.ZoneId - -import org.apache.parquet.VersionParser.ParsedVersion -import org.apache.parquet.column.ColumnDescriptor -import org.apache.parquet.column.page.PageReadStore -import org.apache.parquet.schema.{GroupType, Type} - -import org.apache.spark.sql.execution.datasources.parquet.{ParentContainerUpdater, ParquetRowConverter, ParquetToSparkSchemaConverter, VectorizedColumnReader} -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy -import org.apache.spark.sql.types.StructType - -class ShimParquetRowConverter( - schemaConverter: ParquetToSparkSchemaConverter, - parquetType: GroupType, - catalystType: StructType, - convertTz: Option[ZoneId], - datetimeRebaseMode: String, - int96RebaseMode: String, - int96CDPHive3Compatibility: Boolean, - updater: ParentContainerUpdater -) extends ParquetRowConverter( - schemaConverter, - parquetType, - catalystType, - convertTz, - LegacyBehaviorPolicy.withName(datetimeRebaseMode), - LegacyBehaviorPolicy.withName(int96RebaseMode), - updater) - -class ShimVectorizedColumnReader( - index: Int, - columns: java.util.List[ColumnDescriptor], - types: java.util.List[Type], - pageReadStore: PageReadStore, - convertTz: ZoneId, - datetimeRebaseMode: String, - int96RebaseMode: String, - int96CDPHive3Compatibility: Boolean, - writerVersion: ParsedVersion -) extends VectorizedColumnReader( - columns.get(index), - types.get(index).getOriginalType, - pageReadStore.getPageReader(columns.get(index)), - convertTz, - datetimeRebaseMode, - int96RebaseMode) \ No newline at end of file diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/ShimTrampolineUtil.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/ShimTrampolineUtil.scala deleted file mode 100644 index 87b6183aae1..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/ShimTrampolineUtil.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.rapids.execution - -import org.apache.spark.sql.catalyst.plans.physical.{BroadcastMode, IdentityBroadcastMode} -import org.apache.spark.sql.execution.joins.HashedRelationBroadcastMode -import org.apache.spark.sql.types.{DataType, StructType} - -object ShimTrampolineUtil { - - // unionLikeMerge was only added in Spark 3.2 so be bug compatible and call merge - // https://issues.apache.org/jira/browse/SPARK-36673 - def unionLikeMerge(left: DataType, right: DataType): DataType = - StructType.merge(left, right) - - def isSupportedRelation(mode: BroadcastMode): Boolean = mode match { - case _ : HashedRelationBroadcastMode => true - case IdentityBroadcastMode => true - case _ => false - } -} - diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala deleted file mode 100644 index a6f58095c95..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.rapids.execution.python.shims - -import java.io.DataInputStream -import java.net.Socket -import java.util.concurrent.atomic.AtomicBoolean - -import com.nvidia.spark.rapids.GpuSemaphore - -import org.apache.spark.{SparkEnv, TaskContext} -import org.apache.spark.api.python._ -import org.apache.spark.sql.rapids.execution.python.GpuArrowOutput -import org.apache.spark.sql.vectorized.ColumnarBatch - -/** - * A trait that can be mixed-in with `GpuBasePythonRunner`. It implements the logic from - * Python (Arrow) to GPU/JVM (ColumnarBatch). - */ -trait GpuArrowPythonOutput extends GpuArrowOutput { _: GpuBasePythonRunner[_] => - protected def newReaderIterator( - stream: DataInputStream, - writerThread: WriterThread, - startTime: Long, - env: SparkEnv, - worker: Socket, - releasedOrClosed: AtomicBoolean, - context: TaskContext): Iterator[ColumnarBatch] = { - - new ReaderIterator(stream, writerThread, startTime, env, worker, releasedOrClosed, context) { - val gpuArrowReader = newGpuArrowReader - - protected override def read(): ColumnarBatch = { - if (writerThread.exception.isDefined) { - throw writerThread.exception.get - } - try { - // Because of batching and other things we have to be sure that we release the semaphore - // before any operation that could block. This is because we are using multiple threads - // for a single task and the GpuSemaphore might not wake up both threads associated with - // the task, so a reader can be blocked waiting for data, while a writer is waiting on - // the semaphore - GpuSemaphore.releaseIfNecessary(TaskContext.get()) - if (gpuArrowReader.isStarted && gpuArrowReader.mayHasNext) { - val batch = gpuArrowReader.readNext() - if (batch != null) { - batch - } else { - gpuArrowReader.close() // reach the end, close the reader - read() // read the end signal - } - } else { - stream.readInt() match { - case SpecialLengths.START_ARROW_STREAM => - gpuArrowReader.start(stream) - read() - case SpecialLengths.TIMING_DATA => - handleTimingData() - read() - case SpecialLengths.PYTHON_EXCEPTION_THROWN => - throw handlePythonException() - case SpecialLengths.END_OF_DATA_SECTION => - handleEndOfDataSection() - null - } - } - } catch handleException - } - } - } -} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/AvroUtils.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/AvroUtils.scala deleted file mode 100644 index 881fd81eda0..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/AvroUtils.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.rapids.shims - -import com.nvidia.spark.rapids.RapidsMeta - -import org.apache.spark.sql.avro.AvroOptions - -object AvroUtils { - - def tagSupport( - parsedOptions: AvroOptions, - meta: RapidsMeta[_, _, _]): Unit = { - - } - -} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuJsonToStructsShim.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuJsonToStructsShim.scala deleted file mode 100644 index 6edf6d2049c..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuJsonToStructsShim.scala +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.rapids.shims - -import ai.rapids.cudf.{ColumnVector, ColumnView, DType, Scalar} -import com.nvidia.spark.rapids.{GpuCast, GpuOverrides, RapidsMeta} -import com.nvidia.spark.rapids.Arm.withResource - -import org.apache.spark.sql.catalyst.json.GpuJsonUtils -import org.apache.spark.sql.catalyst.json.JSONOptions -import org.apache.spark.sql.rapids.ExceptionTimeParserPolicy - -object GpuJsonToStructsShim { - def tagDateFormatSupport(meta: RapidsMeta[_, _, _], dateFormat: Option[String]): Unit = { - dateFormat match { - case None | Some("yyyy-MM-dd") => - case dateFormat => - meta.willNotWorkOnGpu(s"GpuJsonToStructs unsupported dateFormat $dateFormat") - } - } - - def castJsonStringToDate(input: ColumnView, options: JSONOptions): ColumnVector = { - GpuJsonUtils.optionalDateFormatInRead(options) match { - case None | Some("yyyy-MM-dd") => - withResource(Scalar.fromString(" ")) { space => - withResource(input.strip(space)) { trimmed => - GpuCast.castStringToDate(trimmed) - } - } - case other => - // should be unreachable due to GpuOverrides checks - throw new IllegalStateException(s"Unsupported dateFormat $other") - } - } - - def tagDateFormatSupportFromScan(meta: RapidsMeta[_, _, _], dateFormat: Option[String]): Unit = { - tagDateFormatSupport(meta, dateFormat) - } - - def castJsonStringToDateFromScan(input: ColumnView, dt: DType, - dateFormat: Option[String]): ColumnVector = { - dateFormat match { - case None | Some("yyyy-MM-dd") => - withResource(input.strip()) { trimmed => - GpuCast.castStringToDateAnsi(trimmed, ansiMode = - GpuOverrides.getTimeParserPolicy == ExceptionTimeParserPolicy) - } - case other => - // should be unreachable due to GpuOverrides checks - throw new IllegalStateException(s"Unsupported dateFormat $other") - } - } - - - def castJsonStringToTimestamp(input: ColumnView, - options: JSONOptions): ColumnVector = { - withResource(Scalar.fromString(" ")) { space => - withResource(input.strip(space)) { trimmed => - // from_json doesn't respect ansi mode - GpuCast.castStringToTimestamp(trimmed, ansiMode = false) - } - } - } -} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala deleted file mode 100644 index 7fa269db71a..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsErrorUtils.scala +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.rapids.shims - -import org.apache.spark.sql.AnalysisException -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.trees.Origin -import org.apache.spark.sql.types.{DataType, Decimal, DecimalType} - -object RapidsErrorUtils extends RapidsQueryErrorUtils { - def invalidArrayIndexError(index: Int, numElements: Int, - isElementAtF: Boolean = false): ArrayIndexOutOfBoundsException = { - // Follow the Spark string format before 3.3.0 - new ArrayIndexOutOfBoundsException(s"Invalid index: $index, numElements: $numElements") - } - - def mapKeyNotExistError( - key: String, - keyType: DataType, - origin: Origin): NoSuchElementException = { - // Follow the Spark string format before 3.3.0 - new NoSuchElementException(s"Key $key does not exist.") - } - - def sqlArrayIndexNotStartAtOneError(): RuntimeException = { - new ArrayIndexOutOfBoundsException("SQL array indices start at 1") - } - - def divByZeroError(origin: Origin): ArithmeticException = { - new ArithmeticException("divide by zero") - } - - def divOverflowError(origin: Origin): ArithmeticException = { - new ArithmeticException("Overflow in integral divide.") - } - - def arithmeticOverflowError( - message: String, - hint: String = "", - errorContext: String = ""): ArithmeticException = { - new ArithmeticException(message) - } - - def cannotChangeDecimalPrecisionError( - value: Decimal, - toType: DecimalType, - context: String = ""): ArithmeticException = { - new ArithmeticException(s"${value.toDebugString} cannot be represented as " + - s"Decimal(${toType.precision}, ${toType.scale}).") - } - - def overflowInIntegralDivideError(context: String = ""): ArithmeticException = { - new ArithmeticException("Overflow in integral divide.") - } - - def foundDuplicateFieldInCaseInsensitiveModeError( - requiredFieldName: String, matchedFields: String): Throwable = { - new RuntimeException(s"""Found duplicate field(s) "$requiredFieldName": """ + - s"$matchedFields in case-insensitive mode") - } - - def tableIdentifierExistsError(tableIdentifier: TableIdentifier): Throwable = { - throw new AnalysisException(s"$tableIdentifier already exists.") - } -} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsQueryErrorUtils.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsQueryErrorUtils.scala deleted file mode 100644 index 266cb4ef54f..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsQueryErrorUtils.scala +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ - -package org.apache.spark.sql.rapids.shims - -import org.apache.hadoop.fs.Path -import org.apache.hadoop.hive.ql.ErrorMsg - -import org.apache.spark.sql.AnalysisException -import org.apache.spark.sql.rapids.execution.RapidsAnalysisException -import org.apache.spark.sql.types.StructType - -trait RapidsQueryErrorUtils { - - def outputPathAlreadyExistsError(qualifiedOutputPath: Path): Throwable = { - new AnalysisException(s"path $qualifiedOutputPath already exists.") - } - - def createTableAsSelectWithNonEmptyDirectoryError(tablePath: String, conf: String): Throwable = { - new AnalysisException(s"CREATE-TABLE-AS-SELECT cannot create table with location to a " + - s"non-empty directory $tablePath. To allow overwriting the existing non-empty directory, " + - s"set '$conf' to true.") - } - - def cannotResolveAttributeError(name: String, outputStr: String): Throwable = { - new AnalysisException(s"Unable to resolve $name given [$outputStr]") - } - - def partitionColumnNotSpecifiedError(format: String, partitionColumn: String): Throwable = { - new AnalysisException(s"Failed to resolve the schema for $format for the partition column: " + - s"$partitionColumn. It must be specified manually.") - } - - def dataSchemaNotSpecifiedError(format: String): Throwable = { - new AnalysisException(s"Unable to infer schema for $format. It must be specified manually.") - } - - def schemaNotSpecifiedForSchemaRelationProviderError(className: String): Throwable = { - new AnalysisException(s"A schema needs to be specified when using $className.") - } - - def userSpecifiedSchemaMismatchActualSchemaError( - schema: StructType, - actualSchema: StructType): Throwable = { - new AnalysisException("The user-specified schema doesn't match the actual schema: " + - s"user-specified: ${schema.toDDL}, actual: ${actualSchema.toDDL}. If " + - "you're using DataFrameReader.schema API or creating a table, please do not " + - "specify the schema. Or if you're scanning an existed table, please drop " + - "it and re-create it.") - } - - def dataSchemaNotSpecifiedError(format: String, fileCatalog: String): Throwable = { - new AnalysisException(s"Unable to infer schema for $format at $fileCatalog. " + - "It must be specified manually") - } - - def invalidDataSourceError(className: String): Throwable = { - new AnalysisException(s"$className is not a valid Spark SQL Data Source.") - } - - def orcNotUsedWithHiveEnabledError(): Throwable = { - new AnalysisException( - s"Hive built-in ORC data source must be used with Hive support enabled. " + - s"Please use the native ORC data source by setting 'spark.sql.orc.impl' to 'native'.") - } - - def failedToFindAvroDataSourceError(provider: String): Throwable = { - new AnalysisException( - s"Failed to find data source: $provider. Avro is built-in but external data " + - "source module since Spark 2.4. Please deploy the application as per " + - "the deployment section of \"Apache Avro Data Source Guide\".") - } - - def failedToFindKafkaDataSourceError(provider: String): Throwable = { - new AnalysisException( - s"Failed to find data source: $provider. Please deploy the application as " + - "per the deployment section of " + - "\"Structured Streaming + Kafka Integration Guide\".") - } - - def findMultipleDataSourceError(provider: String, sourceNames: Seq[String]): Throwable = { - new AnalysisException( - s"Multiple sources found for $provider " + - s"(${sourceNames.mkString(", ")}), please specify the fully qualified class name.") - } - - def dataPathNotExistError(path: String): Throwable = { - new AnalysisException(s"Path does not exist: $path") - } - - def dynamicPartitionParentError: Throwable = { - throw new RapidsAnalysisException(ErrorMsg.PARTITION_DYN_STA_ORDER.getMsg) - } - - def tableOrViewAlreadyExistsError(tableName: String): Throwable = { - new AnalysisException(s"Table $tableName already exists. You need to drop it first.") - } - - def parquetTypeUnsupportedYetError(parquetType: String): Throwable = { - new AnalysisException(s"Parquet type not yet supported: $parquetType.") - } - - def illegalParquetTypeError(parquetType: String): Throwable = { - new AnalysisException(s"Illegal Parquet type: $parquetType.") - } -} \ No newline at end of file diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedReader.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedReader.scala deleted file mode 100644 index f3d88ece9f9..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedReader.scala +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.rapids.shims - -import ai.rapids.cudf.{NvtxColor, NvtxRange} -import com.nvidia.spark.rapids.Arm.withResource - -import org.apache.spark.{MapOutputTracker, SparkEnv, TaskContext} -import org.apache.spark.serializer.SerializerManager -import org.apache.spark.shuffle.ShuffleReadMetricsReporter -import org.apache.spark.sql.rapids.{RapidsShuffleThreadedReaderBase, ShuffleHandleWithMetrics} -import org.apache.spark.storage.BlockManager - -class RapidsShuffleThreadedReader[K, C] ( - startMapIndex: Int, - endMapIndex: Int, - startPartition: Int, - endPartition: Int, - handle: ShuffleHandleWithMetrics[K, C, C], - context: TaskContext, - readMetrics: ShuffleReadMetricsReporter, - maxBytesInFlight: Long, - serializerManager: SerializerManager = SparkEnv.get.serializerManager, - blockManager: BlockManager = SparkEnv.get.blockManager, - mapOutputTracker: MapOutputTracker = SparkEnv.get.mapOutputTracker, - canUseBatchFetch: Boolean = false, - numReaderThreads: Int = 0) - extends RapidsShuffleThreadedReaderBase[K, C]( - handle, - context, - readMetrics, - maxBytesInFlight, - serializerManager = serializerManager, - blockManager = blockManager, - mapOutputTracker = mapOutputTracker, - canUseBatchFetch = canUseBatchFetch, - numReaderThreads = numReaderThreads) { - - override protected def getMapSizes: GetMapSizesResult = { - val shuffleId = handle.shuffleId - val mapSizes = withResource(new NvtxRange("getMapSizesByExecId", NvtxColor.CYAN)) { _ => - mapOutputTracker.getMapSizesByExecutorId( - shuffleId, startMapIndex, endMapIndex, startPartition, endPartition) - } - GetMapSizesResult(mapSizes, canEnableBatchFetch = true) - } -} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedWriter.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedWriter.scala deleted file mode 100644 index a0b4c7f6518..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsShuffleThreadedWriter.scala +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.rapids.shims - -import org.apache.spark.SparkConf -import org.apache.spark.shuffle.ShuffleWriteMetricsReporter -import org.apache.spark.shuffle.api.{ShuffleExecutorComponents, ShuffleMapOutputWriter} -import org.apache.spark.sql.rapids.{RapidsShuffleThreadedWriterBase, ShuffleHandleWithMetrics} -import org.apache.spark.storage.BlockManager - -class RapidsShuffleThreadedWriter[K, V]( - blockManager: BlockManager, - handle: ShuffleHandleWithMetrics[K, V, V], - mapId: Long, - sparkConf: SparkConf, - writeMetrics: ShuffleWriteMetricsReporter, - maxBytesInFlight: Long, - shuffleExecutorComponents: ShuffleExecutorComponents, - numWriterThreads: Int) - extends RapidsShuffleThreadedWriterBase[K, V]( - blockManager, - handle, - mapId, - sparkConf, - writeMetrics, - maxBytesInFlight, - shuffleExecutorComponents, - numWriterThreads) { - - // emptyChecksums: unused in versions of Spark before 3.2.0 - override def doCommitAllPartitions( - writer: ShuffleMapOutputWriter, emptyChecksums: Boolean): Array[Long] = { - writer.commitAllPartitions().getPartitionLengths - } -} - diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/datetimeExpressions.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/datetimeExpressions.scala deleted file mode 100644 index 8a37bd63ca5..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/datetimeExpressions.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.rapids.shims - -import ai.rapids.cudf.{ColumnVector, ColumnView, Scalar} - -import org.apache.spark.sql.catalyst.expressions.{Expression, TimeZoneAwareExpression} -import org.apache.spark.sql.rapids.GpuTimeMath - -case class GpuTimeAdd(start: Expression, - interval: Expression, - timeZoneId: Option[String] = None) - extends GpuTimeMath(start, interval, timeZoneId) { - - override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression = { - copy(timeZoneId = Option(timeZoneId)) - } - - override def intervalMath(us_s: Scalar, us: ColumnView): ColumnVector = { - us.add(us_s) - } -} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/types/shims/PartitionValueCastShims.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/types/shims/PartitionValueCastShims.scala deleted file mode 100644 index fc1b4365dad..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/types/shims/PartitionValueCastShims.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.sql.types.shims - -import java.time.ZoneId - -import org.apache.spark.sql.types.DataType - -object PartitionValueCastShims { - // AnyTimestamp, TimestampNTZTtpe and AnsiIntervalType types are not defined before Spark 3.2.0 - // return false between 311 until 320 - def isSupportedType(dt: DataType): Boolean = false - - def castTo(desiredType: DataType, value: String, zoneId: ZoneId): Any = { - throw new IllegalArgumentException(s"Unexpected type $desiredType") - } -} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/storage/RapidsShuffleBlockFetcherIterator.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/storage/RapidsShuffleBlockFetcherIterator.scala deleted file mode 100644 index b66ca993402..00000000000 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/storage/RapidsShuffleBlockFetcherIterator.scala +++ /dev/null @@ -1,1050 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package org.apache.spark.storage - -import java.io.{InputStream, IOException} -import java.nio.channels.ClosedByInterruptException -import java.util.concurrent.{LinkedBlockingQueue, TimeUnit} -import javax.annotation.concurrent.GuardedBy - -import scala.collection.mutable -import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, LinkedHashMap, Queue} -import scala.util.{Failure, Success} - -import org.apache.commons.io.IOUtils - -import org.apache.spark.{SparkEnv, SparkException, TaskContext} -import org.apache.spark.internal.{config, Logging} -import org.apache.spark.network.buffer.{FileSegmentManagedBuffer, ManagedBuffer} -import org.apache.spark.network.shuffle._ -import org.apache.spark.network.util.TransportConf -import org.apache.spark.serializer.SerializerManager -import org.apache.spark.shuffle.{FetchFailedException, ShuffleReadMetricsReporter} -import org.apache.spark.storage.RapidsShuffleBlockFetcherIterator.SuccessFetchResult -import org.apache.spark.util.{CompletionIterator, TaskCompletionListener, Utils} - -/** - * Taken mostly verbatim from `ShuffleBlockFetcherIterator` except for - * a change to the ownership of `currentResult` (which contains the netty buffer). - * Instead of this iterator owning the result and clearing it on `next`, the - * `BufferReleasingInputStream` is in charge of that. This allows for multiple threads - * to consume different `BufferReleasingInputStream`s produced from this single iterator. - * - * Compare to https://github.com/apache/spark/blob/branch-3.1: - * ./core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala - */ - -/** - * An iterator that fetches multiple blocks. For local blocks, it fetches from the local block - * manager. For remote blocks, it fetches them using the provided BlockTransferService. - * - * This creates an iterator of (BlockID, InputStream) tuples so the caller can handle blocks - * in a pipelined fashion as they are received. - * - * The implementation throttles the remote fetches so they don't exceed maxBytesInFlight to avoid - * using too much memory. - * - * @param context [[TaskContext]], used for metrics update - * @param shuffleClient [[BlockStoreClient]] for fetching remote blocks - * @param blockManager [[BlockManager]] for reading local blocks - * @param blocksByAddress list of blocks to fetch grouped by the [[BlockManagerId]]. - * For each block we also require two info: 1. the size (in bytes as a long - * field) in order to throttle the memory usage; 2. the mapIndex for this - * block, which indicate the index in the map stage. - * Note that zero-sized blocks are already excluded, which happened in - * [[org.apache.spark.MapOutputTracker.convertMapStatuses]]. - * @param streamWrapper A function to wrap the returned input stream. - * @param maxBytesInFlight max size (in bytes) of remote blocks to fetch at any given point. - * @param maxReqsInFlight max number of remote requests to fetch blocks at any given point. - * @param maxBlocksInFlightPerAddress max number of shuffle blocks being fetched at any given point - * for a given remote host:port. - * @param maxReqSizeShuffleToMem max size (in bytes) of a request that can be shuffled to memory. - * @param detectCorrupt whether to detect any corruption in fetched blocks. - * @param shuffleMetrics used to report shuffle metrics. - * @param doBatchFetch fetch continuous shuffle blocks from same executor in batch if the server - * side supports. - */ -private[spark] -final class RapidsShuffleBlockFetcherIterator( - context: TaskContext, - shuffleClient: BlockStoreClient, - blockManager: BlockManager, - blocksByAddress: Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])], - streamWrapper: (BlockId, InputStream) => InputStream, - maxBytesInFlight: Long, - maxReqsInFlight: Int, - maxBlocksInFlightPerAddress: Int, - maxReqSizeShuffleToMem: Long, - detectCorrupt: Boolean, - detectCorruptUseExtraMemory: Boolean, - shuffleMetrics: ShuffleReadMetricsReporter, - doBatchFetch: Boolean) - extends Iterator[(BlockId, InputStream)] with DownloadFileManager with Logging { - - import RapidsShuffleBlockFetcherIterator._ - - // Make remote requests at most maxBytesInFlight / 5 in length; the reason to keep them - // smaller than maxBytesInFlight is to allow multiple, parallel fetches from up to 5 - // nodes, rather than blocking on reading output from one node. - private val targetRemoteRequestSize = math.max(maxBytesInFlight / 5, 1L) - - /** - * Total number of blocks to fetch. - */ - private[this] var numBlocksToFetch = 0 - - /** - * The number of blocks processed by the caller. The iterator is exhausted when - * [[numBlocksProcessed]] == [[numBlocksToFetch]]. - */ - private[this] var numBlocksProcessed = 0 - - private[this] val startTimeNs = System.nanoTime() - - /** Local blocks to fetch, excluding zero-sized blocks. */ - private[this] val localBlocks = scala.collection.mutable.LinkedHashSet[(BlockId, Int)]() - - /** Host local blockIds to fetch by executors, excluding zero-sized blocks. */ - private[this] val hostLocalBlocksByExecutor = - LinkedHashMap[BlockManagerId, Seq[(BlockId, Long, Int)]]() - - /** Host local blocks to fetch, excluding zero-sized blocks. */ - private[this] val hostLocalBlocks = scala.collection.mutable.LinkedHashSet[(BlockId, Int)]() - - /** - * A queue to hold our results. This turns the asynchronous model provided by - * [[org.apache.spark.network.BlockTransferService]] into a synchronous model (iterator). - */ - private[this] val results = new LinkedBlockingQueue[FetchResult] - - /** - * Queue of fetch requests to issue; we'll pull requests off this gradually to make sure that - * the number of bytes in flight is limited to maxBytesInFlight. - */ - private[this] val fetchRequests = new Queue[FetchRequest] - - /** - * Queue of fetch requests which could not be issued the first time they were dequeued. These - * requests are tried again when the fetch constraints are satisfied. - */ - private[this] val deferredFetchRequests = new HashMap[BlockManagerId, Queue[FetchRequest]]() - - /** Current bytes in flight from our requests */ - private[this] var bytesInFlight = 0L - - /** Current number of requests in flight */ - private[this] var reqsInFlight = 0 - - /** Current number of blocks in flight per host:port */ - private[this] val numBlocksInFlightPerAddress = new HashMap[BlockManagerId, Int]() - - /** - * The blocks that can't be decompressed successfully, it is used to guarantee that we retry - * at most once for those corrupted blocks. - */ - private[this] val corruptedBlocks = mutable.HashSet[BlockId]() - - /** - * Whether the iterator is still active. If isZombie is true, the callback interface will no - * longer place fetched blocks into [[results]]. - */ - @GuardedBy("this") - private[this] var isZombie = false - - /** - * A set to store the files used for shuffling remote huge blocks. Files in this set will be - * deleted when cleanup. This is a layer of defensiveness against disk file leaks. - */ - @GuardedBy("this") - private[this] val shuffleFilesSet = mutable.HashSet[DownloadFile]() - - private[this] val onCompleteCallback = new RapidsShuffleFetchCompletionListener(this) - - initialize() - - def resultCount: Int = results.size() - - override def createTempFile(transportConf: TransportConf): DownloadFile = { - // we never need to do any encryption or decryption here, regardless of configs, because that - // is handled at another layer in the code. When encryption is enabled, shuffle data is written - // to disk encrypted in the first place, and sent over the network still encrypted. - new SimpleDownloadFile( - blockManager.diskBlockManager.createTempLocalBlock()._2, transportConf) - } - - override def registerTempFileToClean(file: DownloadFile): Boolean = synchronized { - if (isZombie) { - false - } else { - shuffleFilesSet += file - true - } - } - - /** - * Mark the iterator as zombie, and release all buffers that haven't been deserialized yet. - */ - private[storage] def cleanup(): Unit = { - synchronized { - isZombie = true - } - // Release buffers in the results queue - val iter = results.iterator() - while (iter.hasNext) { - val result = iter.next() - result match { - case SuccessFetchResult(blockId, mapIndex, address, _, buf, _) => - if (address != blockManager.blockManagerId) { - if (hostLocalBlocks.contains(blockId -> mapIndex)) { - shuffleMetrics.incLocalBlocksFetched(1) - shuffleMetrics.incLocalBytesRead(buf.size) - } else { - shuffleMetrics.incRemoteBytesRead(buf.size) - if (buf.isInstanceOf[FileSegmentManagedBuffer]) { - shuffleMetrics.incRemoteBytesReadToDisk(buf.size) - } - shuffleMetrics.incRemoteBlocksFetched(1) - } - } - buf.release() - case _ => - } - } - shuffleFilesSet.foreach { file => - if (!file.delete()) { - logWarning("Failed to cleanup shuffle fetch temp file " + file.path()) - } - } - } - - private[this] def sendRequest(req: FetchRequest): Unit = { - logDebug("Sending request for %d blocks (%s) from %s".format( - req.blocks.size, Utils.bytesToString(req.size), req.address.hostPort)) - bytesInFlight += req.size - reqsInFlight += 1 - - // so we can look up the block info of each blockID - val infoMap = req.blocks.map { - case FetchBlockInfo(blockId, size, mapIndex) => (blockId.toString, (size, mapIndex)) - }.toMap - val remainingBlocks = new HashSet[String]() ++= infoMap.keys - val blockIds = req.blocks.map(_.blockId.toString) - val address = req.address - - val blockFetchingListener = new BlockFetchingListener { - override def onBlockFetchSuccess(blockId: String, buf: ManagedBuffer): Unit = { - // Only add the buffer to results queue if the iterator is not zombie, - // i.e. cleanup() has not been called yet. - RapidsShuffleBlockFetcherIterator.this.synchronized { - if (!isZombie) { - // Increment the ref count because we need to pass this to a different thread. - // This needs to be released after use. - buf.retain() - remainingBlocks -= blockId - results.put(new SuccessFetchResult(BlockId(blockId), infoMap(blockId)._2, - address, infoMap(blockId)._1, buf, remainingBlocks.isEmpty)) - logDebug("remainingBlocks: " + remainingBlocks) - } - } - logTrace(s"Got remote block $blockId after ${Utils.getUsedTimeNs(startTimeNs)}") - } - - override def onBlockFetchFailure(blockId: String, e: Throwable): Unit = { - logError(s"Failed to get block(s) from ${req.address.host}:${req.address.port}", e) - results.put(new FailureFetchResult(BlockId(blockId), infoMap(blockId)._2, address, e)) - } - } - - // Fetch remote shuffle blocks to disk when the request is too large. Since the shuffle data is - // already encrypted and compressed over the wire(w.r.t. the related configs), we can just fetch - // the data and write it to file directly. - if (req.size > maxReqSizeShuffleToMem) { - shuffleClient.fetchBlocks(address.host, address.port, address.executorId, blockIds.toArray, - blockFetchingListener, this) - } else { - shuffleClient.fetchBlocks(address.host, address.port, address.executorId, blockIds.toArray, - blockFetchingListener, null) - } - } - - private[this] def partitionBlocksByFetchMode(): ArrayBuffer[FetchRequest] = { - logDebug(s"maxBytesInFlight: $maxBytesInFlight, targetRemoteRequestSize: " - + s"$targetRemoteRequestSize, maxBlocksInFlightPerAddress: $maxBlocksInFlightPerAddress") - - // Partition to local, host-local and remote blocks. Remote blocks are further split into - // FetchRequests of size at most maxBytesInFlight in order to limit the amount of data in flight - val collectedRemoteRequests = new ArrayBuffer[FetchRequest] - var localBlockBytes = 0L - var hostLocalBlockBytes = 0L - var remoteBlockBytes = 0L - - val fallback = FallbackStorage.FALLBACK_BLOCK_MANAGER_ID.executorId - for ((address, blockInfos) <- blocksByAddress) { - if (Seq(blockManager.blockManagerId.executorId, fallback).contains(address.executorId)) { - checkBlockSizes(blockInfos) - val mergedBlockInfos = mergeContinuousShuffleBlockIdsIfNeeded( - blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)), doBatchFetch) - numBlocksToFetch += mergedBlockInfos.size - localBlocks ++= mergedBlockInfos.map(info => (info.blockId, info.mapIndex)) - localBlockBytes += mergedBlockInfos.map(_.size).sum - } else if (blockManager.hostLocalDirManager.isDefined && - address.host == blockManager.blockManagerId.host) { - checkBlockSizes(blockInfos) - val mergedBlockInfos = mergeContinuousShuffleBlockIdsIfNeeded( - blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)), doBatchFetch) - numBlocksToFetch += mergedBlockInfos.size - val blocksForAddress = - mergedBlockInfos.map(info => (info.blockId, info.size, info.mapIndex)) - hostLocalBlocksByExecutor += address -> blocksForAddress - hostLocalBlocks ++= blocksForAddress.map(info => (info._1, info._3)) - hostLocalBlockBytes += mergedBlockInfos.map(_.size).sum - } else { - remoteBlockBytes += blockInfos.map(_._2).sum - val (_, timeCost) = Utils.timeTakenMs[Unit] { - collectFetchRequests(address, blockInfos, collectedRemoteRequests) - } - logDebug(s"Collected remote fetch requests for $address in $timeCost ms") - } - } - val numRemoteBlocks = collectedRemoteRequests.map(_.blocks.size).sum - val totalBytes = localBlockBytes + remoteBlockBytes + hostLocalBlockBytes - assert(numBlocksToFetch == localBlocks.size + hostLocalBlocks.size + numRemoteBlocks, - s"The number of non-empty blocks $numBlocksToFetch doesn't equal to the number of local " + - s"blocks ${localBlocks.size} + the number of host-local blocks ${hostLocalBlocks.size} " + - s"+ the number of remote blocks ${numRemoteBlocks}.") - logInfo(s"Getting $numBlocksToFetch (${Utils.bytesToString(totalBytes)}) non-empty blocks " + - s"including ${localBlocks.size} (${Utils.bytesToString(localBlockBytes)}) local and " + - s"${hostLocalBlocks.size} (${Utils.bytesToString(hostLocalBlockBytes)}) " + - s"host-local and $numRemoteBlocks (${Utils.bytesToString(remoteBlockBytes)}) remote blocks") - collectedRemoteRequests - } - - private def createFetchRequest( - blocks: Seq[FetchBlockInfo], - address: BlockManagerId): FetchRequest = { - logDebug(s"Creating fetch request of ${blocks.map(_.size).sum} at $address " - + s"with ${blocks.size} blocks") - FetchRequest(address, blocks) - } - - private def createFetchRequests( - curBlocks: Seq[FetchBlockInfo], - address: BlockManagerId, - isLast: Boolean, - collectedRemoteRequests: ArrayBuffer[FetchRequest]): ArrayBuffer[FetchBlockInfo] = { - val mergedBlocks = mergeContinuousShuffleBlockIdsIfNeeded(curBlocks, doBatchFetch) - numBlocksToFetch += mergedBlocks.size - val retBlocks = new ArrayBuffer[FetchBlockInfo] - if (mergedBlocks.length <= maxBlocksInFlightPerAddress) { - collectedRemoteRequests += createFetchRequest(mergedBlocks, address) - } else { - mergedBlocks.grouped(maxBlocksInFlightPerAddress).foreach { blocks => - if (blocks.length == maxBlocksInFlightPerAddress || isLast) { - collectedRemoteRequests += createFetchRequest(blocks, address) - } else { - // The last group does not exceed `maxBlocksInFlightPerAddress`. Put it back - // to `curBlocks`. - retBlocks ++= blocks - numBlocksToFetch -= blocks.size - } - } - } - retBlocks - } - - private def collectFetchRequests( - address: BlockManagerId, - blockInfos: Seq[(BlockId, Long, Int)], - collectedRemoteRequests: ArrayBuffer[FetchRequest]): Unit = { - val iterator = blockInfos.iterator - var curRequestSize = 0L - var curBlocks = new ArrayBuffer[FetchBlockInfo]() - - while (iterator.hasNext) { - val (blockId, size, mapIndex) = iterator.next() - assertPositiveBlockSize(blockId, size) - curBlocks += FetchBlockInfo(blockId, size, mapIndex) - curRequestSize += size - // For batch fetch, the actual block in flight should count for merged block. - val mayExceedsMaxBlocks = !doBatchFetch && curBlocks.size >= maxBlocksInFlightPerAddress - if (curRequestSize >= targetRemoteRequestSize || mayExceedsMaxBlocks) { - curBlocks = createFetchRequests(curBlocks.toSeq, address, isLast = false, - collectedRemoteRequests) - curRequestSize = curBlocks.map(_.size).sum - } - } - // Add in the final request - if (curBlocks.nonEmpty) { - createFetchRequests(curBlocks.toSeq, address, isLast = true, collectedRemoteRequests) - } - } - - private def assertPositiveBlockSize(blockId: BlockId, blockSize: Long): Unit = { - if (blockSize < 0) { - throw BlockException(blockId, "Negative block size " + size) - } else if (blockSize == 0) { - throw BlockException(blockId, "Zero-sized blocks should be excluded.") - } - } - - private def checkBlockSizes(blockInfos: Seq[(BlockId, Long, Int)]): Unit = { - blockInfos.foreach { case (blockId, size, _) => assertPositiveBlockSize(blockId, size) } - } - - /** - * Fetch the local blocks while we are fetching remote blocks. This is ok because - * `ManagedBuffer`'s memory is allocated lazily when we create the input stream, so all we - * track in-memory are the ManagedBuffer references themselves. - */ - private[this] def fetchLocalBlocks(): Unit = { - logDebug(s"Start fetching local blocks: ${localBlocks.mkString(", ")}") - val iter = localBlocks.iterator - while (iter.hasNext) { - val (blockId, mapIndex) = iter.next() - try { - val buf = blockManager.getLocalBlockData(blockId) - shuffleMetrics.incLocalBlocksFetched(1) - shuffleMetrics.incLocalBytesRead(buf.size) - buf.retain() - results.put(new SuccessFetchResult(blockId, mapIndex, blockManager.blockManagerId, - buf.size(), buf, false)) - } catch { - // If we see an exception, stop immediately. - case e: Exception => - e match { - // ClosedByInterruptException is an excepted exception when kill task, - // don't log the exception stack trace to avoid confusing users. - // See: SPARK-28340 - case ce: ClosedByInterruptException => - logError("Error occurred while fetching local blocks, " + ce.getMessage) - case ex: Exception => logError("Error occurred while fetching local blocks", ex) - } - results.put(new FailureFetchResult(blockId, mapIndex, blockManager.blockManagerId, e)) - return - } - } - } - - private[this] def fetchHostLocalBlock( - blockId: BlockId, - mapIndex: Int, - localDirs: Array[String], - blockManagerId: BlockManagerId): Boolean = { - try { - val buf = blockManager.getHostLocalShuffleData(blockId, localDirs) - buf.retain() - results.put(SuccessFetchResult(blockId, mapIndex, blockManagerId, buf.size(), buf, - isNetworkReqDone = false)) - true - } catch { - case e: Exception => - // If we see an exception, stop immediately. - logError(s"Error occurred while fetching local blocks", e) - results.put(FailureFetchResult(blockId, mapIndex, blockManagerId, e)) - false - } - } - - /** - * Fetch the host-local blocks while we are fetching remote blocks. This is ok because - * `ManagedBuffer`'s memory is allocated lazily when we create the input stream, so all we - * track in-memory are the ManagedBuffer references themselves. - */ - private[this] def fetchHostLocalBlocks(hostLocalDirManager: HostLocalDirManager): Unit = { - val cachedDirsByExec = hostLocalDirManager.getCachedHostLocalDirs - val (hostLocalBlocksWithCachedDirs, hostLocalBlocksWithMissingDirs) = { - val (hasCache, noCache) = hostLocalBlocksByExecutor.partition { case (hostLocalBmId, _) => - cachedDirsByExec.contains(hostLocalBmId.executorId) - } - (hasCache.toMap, noCache.toMap) - } - - if (hostLocalBlocksWithMissingDirs.nonEmpty) { - logDebug(s"Asynchronous fetching host-local blocks without cached executors' dir: " + - s"${hostLocalBlocksWithMissingDirs.mkString(", ")}") - - // If the external shuffle service is enabled, we'll fetch the local directories for - // multiple executors from the external shuffle service, which located at the same host - // with the executors, in once. Otherwise, we'll fetch the local directories from those - // executors directly one by one. The fetch requests won't be too much since one host is - // almost impossible to have many executors at the same time practically. - val dirFetchRequests = if (blockManager.externalShuffleServiceEnabled) { - val host = blockManager.blockManagerId.host - val port = blockManager.externalShuffleServicePort - Seq((host, port, hostLocalBlocksWithMissingDirs.keys.toArray)) - } else { - hostLocalBlocksWithMissingDirs.keys.map(bmId => (bmId.host, bmId.port, Array(bmId))).toSeq - } - - dirFetchRequests.foreach { case (host, port, bmIds) => - hostLocalDirManager.getHostLocalDirs(host, port, bmIds.map(_.executorId)) { - case Success(dirsByExecId) => - fetchMultipleHostLocalBlocks( - hostLocalBlocksWithMissingDirs.filterKeys(bmIds.contains).toMap, - dirsByExecId, - cached = false) - - case Failure(throwable) => - logError("Error occurred while fetching host local blocks", throwable) - val bmId = bmIds.head - val blockInfoSeq = hostLocalBlocksWithMissingDirs(bmId) - val (blockId, _, mapIndex) = blockInfoSeq.head - results.put(FailureFetchResult(blockId, mapIndex, bmId, throwable)) - } - } - } - - if (hostLocalBlocksWithCachedDirs.nonEmpty) { - logDebug(s"Synchronous fetching host-local blocks with cached executors' dir: " + - s"${hostLocalBlocksWithCachedDirs.mkString(", ")}") - fetchMultipleHostLocalBlocks(hostLocalBlocksWithCachedDirs, cachedDirsByExec, cached = true) - } - } - - private def fetchMultipleHostLocalBlocks( - bmIdToBlocks: Map[BlockManagerId, Seq[(BlockId, Long, Int)]], - localDirsByExecId: Map[String, Array[String]], - cached: Boolean): Unit = { - // We use `forall` because once there's a failed block fetch, `fetchHostLocalBlock` will put - // a `FailureFetchResult` immediately to the `results`. So there's no reason to fetch the - // remaining blocks. - val allFetchSucceeded = bmIdToBlocks.forall { case (bmId, blockInfos) => - blockInfos.forall { case (blockId, _, mapIndex) => - fetchHostLocalBlock(blockId, mapIndex, localDirsByExecId(bmId.executorId), bmId) - } - } - if (allFetchSucceeded) { - logDebug(s"Got host-local blocks from ${bmIdToBlocks.keys.mkString(", ")} " + - s"(${if (cached) "with" else "without"} cached executors' dir) " + - s"in ${Utils.getUsedTimeNs(startTimeNs)}") - } - } - - private[this] def initialize(): Unit = { - // Add a task completion callback (called in both success case and failure case) to cleanup. - context.addTaskCompletionListener(onCompleteCallback) - - // Partition blocks by the different fetch modes: local, host-local and remote blocks. - val remoteRequests = partitionBlocksByFetchMode() - // Add the remote requests into our queue in a random order - fetchRequests ++= Utils.randomize(remoteRequests) - assert ((0 == reqsInFlight) == (0 == bytesInFlight), - "expected reqsInFlight = 0 but found reqsInFlight = " + reqsInFlight + - ", expected bytesInFlight = 0 but found bytesInFlight = " + bytesInFlight) - - // Send out initial requests for blocks, up to our maxBytesInFlight - fetchUpToMaxBytes() - - val numFetches = remoteRequests.size - fetchRequests.size - logInfo(s"Started $numFetches remote fetches in ${Utils.getUsedTimeNs(startTimeNs)}") - - // Get Local Blocks - fetchLocalBlocks() - logDebug(s"Got local blocks in ${Utils.getUsedTimeNs(startTimeNs)}") - - if (hostLocalBlocks.nonEmpty) { - blockManager.hostLocalDirManager.foreach(fetchHostLocalBlocks) - } - } - - override def hasNext: Boolean = numBlocksProcessed < numBlocksToFetch - - /** - * Fetches the next (BlockId, InputStream). If a task fails, the ManagedBuffers - * underlying each InputStream will be freed by the cleanup() method registered with the - * TaskCompletionListener. However, callers should close() these InputStreams - * as soon as they are no longer needed, in order to release memory as early as possible. - * - * Throws a FetchFailedException if the next block could not be fetched. - */ - override def next(): (BlockId, InputStream) = { - if (!hasNext) { - throw new NoSuchElementException() - } - - numBlocksProcessed += 1 - - var result: FetchResult = null - var input: InputStream = null - var streamCompressedOrEncrypted: Boolean = false - // Take the next fetched result and try to decompress it to detect data corruption, - // then fetch it one more time if it's corrupt, throw FailureFetchResult if the second fetch - // is also corrupt, so the previous stage could be retried. - // For local shuffle block, throw FailureFetchResult for the first IOException. - while (result == null) { - val startFetchWait = System.nanoTime() - result = results.take() - val fetchWaitTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startFetchWait) - shuffleMetrics.incFetchWaitTime(fetchWaitTime) - - result match { - case SuccessFetchResult(blockId, mapIndex, address, size, buf, isNetworkReqDone) => - if (address != blockManager.blockManagerId) { - if (hostLocalBlocks.contains(blockId -> mapIndex)) { - shuffleMetrics.incLocalBlocksFetched(1) - shuffleMetrics.incLocalBytesRead(buf.size) - } else { - numBlocksInFlightPerAddress(address) = numBlocksInFlightPerAddress(address) - 1 - shuffleMetrics.incRemoteBytesRead(buf.size) - if (buf.isInstanceOf[FileSegmentManagedBuffer]) { - shuffleMetrics.incRemoteBytesReadToDisk(buf.size) - } - shuffleMetrics.incRemoteBlocksFetched(1) - bytesInFlight -= size - } - } - if (isNetworkReqDone) { - reqsInFlight -= 1 - logDebug("Number of requests in flight " + reqsInFlight) - } - - if (buf.size == 0) { - // We will never legitimately receive a zero-size block. All blocks with zero records - // have zero size and all zero-size blocks have no records (and hence should never - // have been requested in the first place). This statement relies on behaviors of the - // shuffle writers, which are guaranteed by the following test cases: - // - // - BypassMergeSortShuffleWriterSuite: "write with some empty partitions" - // - UnsafeShuffleWriterSuite: "writeEmptyIterator" - // - DiskBlockObjectWriterSuite: "commit() and close() without ever opening or writing" - // - // There is not an explicit test for SortShuffleWriter but the underlying APIs that - // uses are shared by the UnsafeShuffleWriter (both writers use DiskBlockObjectWriter - // which returns a zero-size from commitAndGet() in case no records were written - // since the last call. - val msg = s"Received a zero-size buffer for block $blockId from $address " + - s"(expectedApproxSize = $size, isNetworkReqDone=$isNetworkReqDone)" - throwFetchFailedException(blockId, mapIndex, address, new IOException(msg)) - } - - val in = try { - buf.createInputStream() - } catch { - // The exception could only be throwed by local shuffle block - case e: IOException => - assert(buf.isInstanceOf[FileSegmentManagedBuffer]) - e match { - case ce: ClosedByInterruptException => - logError("Failed to create input stream from local block, " + - ce.getMessage) - case e: IOException => logError("Failed to create input stream from local block", e) - } - buf.release() - throwFetchFailedException(blockId, mapIndex, address, e) - } - try { - input = streamWrapper(blockId, in) - // If the stream is compressed or wrapped, then we optionally decompress/unwrap the - // first maxBytesInFlight/3 bytes into memory, to check for corruption in that portion - // of the data. But even if 'detectCorruptUseExtraMemory' configuration is off, or if - // the corruption is later, we'll still detect the corruption later in the stream. - streamCompressedOrEncrypted = !input.eq(in) - if (streamCompressedOrEncrypted && detectCorruptUseExtraMemory) { - // TODO: manage the memory used here, and spill it into disk in case of OOM. - input = Utils.copyStreamUpTo(input, maxBytesInFlight / 3) - } - } catch { - case e: IOException => - buf.release() - if (buf.isInstanceOf[FileSegmentManagedBuffer] - || corruptedBlocks.contains(blockId)) { - throwFetchFailedException(blockId, mapIndex, address, e) - } else { - logWarning(s"got an corrupted block $blockId from $address, fetch again", e) - corruptedBlocks += blockId - fetchRequests += FetchRequest( - address, Array(FetchBlockInfo(blockId, size, mapIndex))) - result = null - } - } finally { - // TODO: release the buf here to free memory earlier - if (input == null) { - // Close the underlying stream if there was an issue in wrapping the stream using - // streamWrapper - in.close() - } - } - - case FailureFetchResult(blockId, mapIndex, address, e) => - throwFetchFailedException(blockId, mapIndex, address, e) - } - - // Send fetch requests up to maxBytesInFlight - fetchUpToMaxBytes() - } - - val currentResult = result.asInstanceOf[SuccessFetchResult] - (currentResult.blockId, - new RapidsBufferReleasingInputStream( - input, - this, - currentResult, - currentResult.blockId, - currentResult.mapIndex, - currentResult.address, - detectCorrupt && streamCompressedOrEncrypted)) - } - - def toCompletionIterator: Iterator[(BlockId, InputStream)] = { - CompletionIterator[(BlockId, InputStream), this.type](this, - onCompleteCallback.onComplete(context)) - } - - private def fetchUpToMaxBytes(): Unit = { - // Send fetch requests up to maxBytesInFlight. If you cannot fetch from a remote host - // immediately, defer the request until the next time it can be processed. - - // Process any outstanding deferred fetch requests if possible. - if (deferredFetchRequests.nonEmpty) { - for ((remoteAddress, defReqQueue) <- deferredFetchRequests) { - while (isRemoteBlockFetchable(defReqQueue) && - !isRemoteAddressMaxedOut(remoteAddress, defReqQueue.front)) { - val request = defReqQueue.dequeue() - logDebug(s"Processing deferred fetch request for $remoteAddress with " - + s"${request.blocks.length} blocks") - send(remoteAddress, request) - if (defReqQueue.isEmpty) { - deferredFetchRequests -= remoteAddress - } - } - } - } - - // Process any regular fetch requests if possible. - while (isRemoteBlockFetchable(fetchRequests)) { - val request = fetchRequests.dequeue() - val remoteAddress = request.address - if (isRemoteAddressMaxedOut(remoteAddress, request)) { - logDebug(s"Deferring fetch request for $remoteAddress with ${request.blocks.size} blocks") - val defReqQueue = deferredFetchRequests.getOrElse(remoteAddress, new Queue[FetchRequest]()) - defReqQueue.enqueue(request) - deferredFetchRequests(remoteAddress) = defReqQueue - } else { - send(remoteAddress, request) - } - } - - def send(remoteAddress: BlockManagerId, request: FetchRequest): Unit = { - sendRequest(request) - numBlocksInFlightPerAddress(remoteAddress) = - numBlocksInFlightPerAddress.getOrElse(remoteAddress, 0) + request.blocks.size - } - - def isRemoteBlockFetchable(fetchReqQueue: Queue[FetchRequest]): Boolean = { - fetchReqQueue.nonEmpty && - (bytesInFlight == 0 || - (reqsInFlight + 1 <= maxReqsInFlight && - bytesInFlight + fetchReqQueue.front.size <= maxBytesInFlight)) - } - - // Checks if sending a new fetch request will exceed the max no. of blocks being fetched from a - // given remote address. - def isRemoteAddressMaxedOut(remoteAddress: BlockManagerId, request: FetchRequest): Boolean = { - numBlocksInFlightPerAddress.getOrElse(remoteAddress, 0) + request.blocks.size > - maxBlocksInFlightPerAddress - } - } - - private[storage] def throwFetchFailedException( - blockId: BlockId, - mapIndex: Int, - address: BlockManagerId, - e: Throwable) = { - blockId match { - case ShuffleBlockId(shufId, mapId, reduceId) => - throw new FetchFailedException(address, shufId, mapId, mapIndex, reduceId, e) - case ShuffleBlockBatchId(shuffleId, mapId, startReduceId, _) => - throw new FetchFailedException(address, shuffleId, mapId, mapIndex, startReduceId, e) - case _ => - throw new SparkException( - "Failed to get block " + blockId + ", which is not a shuffle block", e) - } - } -} - -/** - * Helper class that ensures a ManagedBuffer is released upon InputStream.close() and - * also detects stream corruption if streamCompressedOrEncrypted is true - */ -private class RapidsBufferReleasingInputStream( - // This is visible for testing - private[storage] val delegate: InputStream, - private val iterator: RapidsShuffleBlockFetcherIterator, - private val currentResult: SuccessFetchResult, - private val blockId: BlockId, - private val mapIndex: Int, - private val address: BlockManagerId, - private val detectCorruption: Boolean) - extends InputStream { - private[this] var closed = false - - @scala.annotation.nowarn("msg=method closeQuietly in class IOUtils is deprecated") - override def read(): Int = { - try { - delegate.read() - } catch { - case e: IOException if detectCorruption => - IOUtils.closeQuietly(this) - iterator.throwFetchFailedException(blockId, mapIndex, address, e) - } - } - - override def close(): Unit = { - if (!closed) { - delegate.close() - currentResult.buf.release() - closed = true - } - } - - override def available(): Int = delegate.available() - - override def mark(readlimit: Int): Unit = delegate.mark(readlimit) - - @scala.annotation.nowarn("msg=method closeQuietly in class IOUtils is deprecated") - override def skip(n: Long): Long = { - try { - delegate.skip(n) - } catch { - case e: IOException if detectCorruption => - IOUtils.closeQuietly(this) - iterator.throwFetchFailedException(blockId, mapIndex, address, e) - } - } - - override def markSupported(): Boolean = delegate.markSupported() - - @scala.annotation.nowarn("msg=method closeQuietly in class IOUtils is deprecated") - override def read(b: Array[Byte]): Int = { - try { - delegate.read(b) - } catch { - case e: IOException if detectCorruption => - IOUtils.closeQuietly(this) - iterator.throwFetchFailedException(blockId, mapIndex, address, e) - } - } - - @scala.annotation.nowarn("msg=method closeQuietly in class IOUtils is deprecated") - override def read(b: Array[Byte], off: Int, len: Int): Int = { - try { - delegate.read(b, off, len) - } catch { - case e: IOException if detectCorruption => - IOUtils.closeQuietly(this) - iterator.throwFetchFailedException(blockId, mapIndex, address, e) - } - } - - override def reset(): Unit = delegate.reset() -} - -/** - * A listener to be called at the completion of the ShuffleBlockFetcherIterator - * @param data the ShuffleBlockFetcherIterator to process - */ -private class RapidsShuffleFetchCompletionListener(var data: RapidsShuffleBlockFetcherIterator) - extends TaskCompletionListener { - - override def onTaskCompletion(context: TaskContext): Unit = { - if (data != null) { - data.cleanup() - // Null out the referent here to make sure we don't keep a reference to this - // ShuffleBlockFetcherIterator, after we're done reading from it, to let it be - // collected during GC. Otherwise we can hold metadata on block locations(blocksByAddress) - data = null - } - } - - // Just an alias for onTaskCompletion to avoid confusing - def onComplete(context: TaskContext): Unit = this.onTaskCompletion(context) -} - -object RapidsShuffleBlockFetcherIterator { - /** - * This function is used to merged blocks when doBatchFetch is true. Blocks which have the - * same `mapId` can be merged into one block batch. The block batch is specified by a range - * of reduceId, which implies the continuous shuffle blocks that we can fetch in a batch. - * For example, input blocks like (shuffle_0_0_0, shuffle_0_0_1, shuffle_0_1_0) can be - * merged into (shuffle_0_0_0_2, shuffle_0_1_0_1), and input blocks like (shuffle_0_0_0_2, - * shuffle_0_0_2, shuffle_0_0_3) can be merged into (shuffle_0_0_0_4). - * - * @param blocks blocks to be merged if possible. May contains already merged blocks. - * @param doBatchFetch whether to merge blocks. - * @return the input blocks if doBatchFetch=false, or the merged blocks if doBatchFetch=true. - */ - def mergeContinuousShuffleBlockIdsIfNeeded( - blocks: Seq[FetchBlockInfo], - doBatchFetch: Boolean): Seq[FetchBlockInfo] = { - val result = if (doBatchFetch) { - val curBlocks = new ArrayBuffer[FetchBlockInfo] - val mergedBlockInfo = new ArrayBuffer[FetchBlockInfo] - - def mergeFetchBlockInfo(toBeMerged: ArrayBuffer[FetchBlockInfo]): FetchBlockInfo = { - val startBlockId = toBeMerged.head.blockId.asInstanceOf[ShuffleBlockId] - - // The last merged block may comes from the input, and we can merge more blocks - // into it, if the map id is the same. - def shouldMergeIntoPreviousBatchBlockId = - mergedBlockInfo.last.blockId.asInstanceOf[ShuffleBlockBatchId].mapId == startBlockId.mapId - - val (startReduceId, size) = - if (mergedBlockInfo.nonEmpty && shouldMergeIntoPreviousBatchBlockId) { - // Remove the previous batch block id as we will add a new one to replace it. - val removed = mergedBlockInfo.remove(mergedBlockInfo.length - 1) - (removed.blockId.asInstanceOf[ShuffleBlockBatchId].startReduceId, - removed.size + toBeMerged.map(_.size).sum) - } else { - (startBlockId.reduceId, toBeMerged.map(_.size).sum) - } - - FetchBlockInfo( - ShuffleBlockBatchId( - startBlockId.shuffleId, - startBlockId.mapId, - startReduceId, - toBeMerged.last.blockId.asInstanceOf[ShuffleBlockId].reduceId + 1), - size, - toBeMerged.head.mapIndex) - } - - val iter = blocks.iterator - while (iter.hasNext) { - val info = iter.next() - // It's possible that the input block id is already a batch ID. For example, we merge some - // blocks, and then make fetch requests with the merged blocks according to "max blocks per - // request". The last fetch request may be too small, and we give up and put the remaining - // merged blocks back to the input list. - if (info.blockId.isInstanceOf[ShuffleBlockBatchId]) { - mergedBlockInfo += info - } else { - if (curBlocks.isEmpty) { - curBlocks += info - } else { - val curBlockId = info.blockId.asInstanceOf[ShuffleBlockId] - val currentMapId = curBlocks.head.blockId.asInstanceOf[ShuffleBlockId].mapId - if (curBlockId.mapId != currentMapId) { - mergedBlockInfo += mergeFetchBlockInfo(curBlocks) - curBlocks.clear() - } - curBlocks += info - } - } - } - if (curBlocks.nonEmpty) { - mergedBlockInfo += mergeFetchBlockInfo(curBlocks) - } - mergedBlockInfo - } else { - blocks - } - result.toSeq - } - - /** - * The block information to fetch used in FetchRequest. - * @param blockId block id - * @param size estimated size of the block. Note that this is NOT the exact bytes. - * Size of remote block is used to calculate bytesInFlight. - * @param mapIndex the mapIndex for this block, which indicate the index in the map stage. - */ - private[storage] case class FetchBlockInfo( - blockId: BlockId, - size: Long, - mapIndex: Int) - - /** - * A request to fetch blocks from a remote BlockManager. - * @param address remote BlockManager to fetch from. - * @param blocks Sequence of the information for blocks to fetch from the same address. - */ - case class FetchRequest(address: BlockManagerId, blocks: Seq[FetchBlockInfo]) { - val size = blocks.map(_.size).sum - } - - /** - * Result of a fetch from a remote block. - */ - private[storage] sealed trait FetchResult { - val blockId: BlockId - val address: BlockManagerId - } - - /** - * Result of a fetch from a remote block successfully. - * @param blockId block id - * @param mapIndex the mapIndex for this block, which indicate the index in the map stage. - * @param address BlockManager that the block was fetched from. - * @param size estimated size of the block. Note that this is NOT the exact bytes. - * Size of remote block is used to calculate bytesInFlight. - * @param buf `ManagedBuffer` for the content. - * @param isNetworkReqDone Is this the last network request for this host in this fetch request. - */ - private[storage] case class SuccessFetchResult( - blockId: BlockId, - mapIndex: Int, - address: BlockManagerId, - size: Long, - buf: ManagedBuffer, - isNetworkReqDone: Boolean) extends FetchResult { - require(buf != null) - require(size >= 0) - } - - /** - * Result of a fetch from a remote block unsuccessfully. - * @param blockId block id - * @param mapIndex the mapIndex for this block, which indicate the index in the map stage - * @param address BlockManager that the block was attempted to be fetched from - * @param e the failure exception - */ - private[storage] case class FailureFetchResult( - blockId: BlockId, - mapIndex: Int, - address: BlockManagerId, - e: Throwable) - extends FetchResult - - def makeIterator( - context: TaskContext, - blockManager: BlockManager, - sparkEnv: SparkEnv, - blocksByAddress: Iterator[(BlockManagerId, Seq[(BlockId, Long, Int)])], - serializerManager: SerializerManager, - readMetrics: ShuffleReadMetricsReporter, - fetchContinuousBlocksInBatch: Boolean): RapidsShuffleBlockFetcherIterator = { - new RapidsShuffleBlockFetcherIterator( - context, - blockManager.blockStoreClient, - blockManager, - blocksByAddress, - serializerManager.wrapStream, - // Note: we use getSizeAsMb when no suffix is provided for backwards compatibility - sparkEnv.conf.get(config.REDUCER_MAX_SIZE_IN_FLIGHT) * 1024 * 1024, // 48mb default per task - sparkEnv.conf.get(config.REDUCER_MAX_REQS_IN_FLIGHT), //Int.MaxValue by default - sparkEnv.conf.get(config.REDUCER_MAX_BLOCKS_IN_FLIGHT_PER_ADDRESS), - sparkEnv.conf.get(config.MAX_REMOTE_BLOCK_SIZE_FETCH_TO_MEM), - sparkEnv.conf.get(config.SHUFFLE_DETECT_CORRUPT), - sparkEnv.conf.get(config.SHUFFLE_DETECT_CORRUPT_MEMORY), - readMetrics, - fetchContinuousBlocksInBatch) - } -} \ No newline at end of file diff --git a/sql-plugin/src/main/spark312/scala/com/nvidia/spark/rapids/shims/SparkShims.scala b/sql-plugin/src/main/spark312/scala/com/nvidia/spark/rapids/shims/SparkShims.scala deleted file mode 100644 index 673f10fb8e6..00000000000 --- a/sql-plugin/src/main/spark312/scala/com/nvidia/spark/rapids/shims/SparkShims.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "312"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import org.apache.parquet.schema.MessageType - -import org.apache.spark.sql.execution.datasources.parquet.ParquetFilters - -object SparkShimImpl extends Spark31XShims { - override def hasCastFloatTimestampUpcast: Boolean = true - - override def reproduceEmptyStringBug: Boolean = true - - override def getParquetFilters( - schema: MessageType, - pushDownDate: Boolean, - pushDownTimestamp: Boolean, - pushDownDecimal: Boolean, - pushDownStartWith: Boolean, - pushDownInFilterThreshold: Int, - caseSensitive: Boolean, - lookupFileMeta: String => String, - dateTimeRebaseModeFromConf: String): ParquetFilters = { - new ParquetFilters(schema, pushDownDate, pushDownTimestamp, pushDownDecimal, pushDownStartWith, - pushDownInFilterThreshold, caseSensitive) - } -} diff --git a/sql-plugin/src/main/spark312/scala/com/nvidia/spark/rapids/shims/spark312/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark312/scala/com/nvidia/spark/rapids/shims/spark312/SparkShimServiceProvider.scala deleted file mode 100644 index 1454d35e629..00000000000 --- a/sql-plugin/src/main/spark312/scala/com/nvidia/spark/rapids/shims/spark312/SparkShimServiceProvider.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "312"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark312 - -import com.nvidia.spark.rapids.SparkShimVersion - -object SparkShimServiceProvider { - val VERSION = SparkShimVersion(3, 1, 2) - val VERSIONNAMES = Seq(s"$VERSION") -} - -class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { - - override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION - - def matchesVersion(version: String): Boolean = { - SparkShimServiceProvider.VERSIONNAMES.contains(version) - } -} diff --git a/sql-plugin/src/main/spark312/scala/com/nvidia/spark/rapids/spark312/RapidsShuffleManager.scala b/sql-plugin/src/main/spark312/scala/com/nvidia/spark/rapids/spark312/RapidsShuffleManager.scala deleted file mode 100644 index a33e56e05a8..00000000000 --- a/sql-plugin/src/main/spark312/scala/com/nvidia/spark/rapids/spark312/RapidsShuffleManager.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "312"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.spark312 - -import org.apache.spark.SparkConf -import org.apache.spark.sql.rapids.ProxyRapidsShuffleInternalManagerBase - -/** A shuffle manager optimized for the RAPIDS Plugin for Apache Spark. */ -sealed class RapidsShuffleManager( - conf: SparkConf, - isDriver: Boolean -) extends ProxyRapidsShuffleInternalManagerBase(conf, isDriver) - diff --git a/sql-plugin/src/main/spark313/scala/com/nvidia/spark/rapids/shims/SparkShims.scala b/sql-plugin/src/main/spark313/scala/com/nvidia/spark/rapids/shims/SparkShims.scala deleted file mode 100644 index f28619b22d2..00000000000 --- a/sql-plugin/src/main/spark313/scala/com/nvidia/spark/rapids/shims/SparkShims.scala +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims - -import org.apache.parquet.schema.MessageType - -import org.apache.spark.sql.execution.datasources.DataSourceUtils -import org.apache.spark.sql.execution.datasources.parquet.ParquetFilters - -object SparkShimImpl extends Spark31XShims { - override def getParquetFilters( - schema: MessageType, - pushDownDate: Boolean, - pushDownTimestamp: Boolean, - pushDownDecimal: Boolean, - pushDownStartWith: Boolean, - pushDownInFilterThreshold: Int, - caseSensitive: Boolean, - lookupFileMeta: String => String, - dateTimeRebaseModeFromConf: String): ParquetFilters = { - val datetimeRebaseMode = DataSourceUtils - .datetimeRebaseMode(lookupFileMeta, dateTimeRebaseModeFromConf) - new ParquetFilters(schema, pushDownDate, pushDownTimestamp, pushDownDecimal, pushDownStartWith, - pushDownInFilterThreshold, caseSensitive, datetimeRebaseMode) - } - - override def hasCastFloatTimestampUpcast: Boolean = true - - override def isCastingStringToNegDecimalScaleSupported: Boolean = true - - override def reproduceEmptyStringBug: Boolean = true -} diff --git a/sql-plugin/src/main/spark313/scala/com/nvidia/spark/rapids/shims/spark313/SparkShimServiceProvider.scala b/sql-plugin/src/main/spark313/scala/com/nvidia/spark/rapids/shims/spark313/SparkShimServiceProvider.scala deleted file mode 100644 index 9dafcdde8c6..00000000000 --- a/sql-plugin/src/main/spark313/scala/com/nvidia/spark/rapids/shims/spark313/SparkShimServiceProvider.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark313 - -import com.nvidia.spark.rapids.SparkShimVersion - -object SparkShimServiceProvider { - val VERSION = SparkShimVersion(3, 1, 3) - val VERSIONNAMES = Seq(s"$VERSION") -} - -class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { - - override def getShimVersion: SparkShimVersion = SparkShimServiceProvider.VERSION - - def matchesVersion(version: String): Boolean = { - SparkShimServiceProvider.VERSIONNAMES.contains(version) - } -} diff --git a/sql-plugin/src/main/spark313/scala/com/nvidia/spark/rapids/spark313/RapidsShuffleManager.scala b/sql-plugin/src/main/spark313/scala/com/nvidia/spark/rapids/spark313/RapidsShuffleManager.scala deleted file mode 100644 index 3cad33ed523..00000000000 --- a/sql-plugin/src/main/spark313/scala/com/nvidia/spark/rapids/spark313/RapidsShuffleManager.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.spark313 - -import org.apache.spark.SparkConf -import org.apache.spark.sql.rapids.ProxyRapidsShuffleInternalManagerBase - -/** A shuffle manager optimized for the RAPIDS Plugin for Apache Spark. */ -sealed class RapidsShuffleManager( - conf: SparkConf, - isDriver: Boolean -) extends ProxyRapidsShuffleInternalManagerBase(conf, isDriver) diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AQEUtils.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/AQEUtils.scala similarity index 97% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AQEUtils.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/AQEUtils.scala index 9535bd4caf9..5c17d43b463 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AQEUtils.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/AQEUtils.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala index d024f58d152..59cc47736f9 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AnsiCastShim.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/AnsiCastShim.scala similarity index 97% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AnsiCastShim.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/AnsiCastShim.scala index 3250688c666..c6477489a92 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AnsiCastShim.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/AnsiCastShim.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AnsiUtil.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/AnsiUtil.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AnsiUtil.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/AnsiUtil.scala index 623bb80e38f..95260042742 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AnsiUtil.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/AnsiUtil.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BloomFilterShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/BloomFilterShims.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BloomFilterShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/BloomFilterShims.scala index 93bebf104fe..a0db9f104af 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BloomFilterShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/BloomFilterShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala index 40ea1d4145a..499251aa32f 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/BucketSpecForHiveShim.scala @@ -14,9 +14,6 @@ * limitations under the License. */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BucketingUtilsShim.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/BucketingUtilsShim.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BucketingUtilsShim.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/BucketingUtilsShim.scala index 835464ab88b..a605edb8d85 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BucketingUtilsShim.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/BucketingUtilsShim.scala @@ -14,9 +14,6 @@ * limitations under the License. */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CastCheckShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CastCheckShims.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CastCheckShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CastCheckShims.scala index 64d6ec26602..5025f999b3b 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CastCheckShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CastCheckShims.scala @@ -16,9 +16,6 @@ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CastingConfigShim.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CastingConfigShim.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CastingConfigShim.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CastingConfigShim.scala index c3703cc81ea..948f62d6297 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CastingConfigShim.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CastingConfigShim.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CharVarcharUtilsShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CharVarcharUtilsShims.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CharVarcharUtilsShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CharVarcharUtilsShims.scala index bd611edc5e2..1ed48e5e4fc 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CharVarcharUtilsShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CharVarcharUtilsShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ColumnDefaultValuesShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ColumnDefaultValuesShims.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ColumnDefaultValuesShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ColumnDefaultValuesShims.scala index f2ce2608cce..8815103e630 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ColumnDefaultValuesShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ColumnDefaultValuesShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala index d1a26dc80fc..c5d27c1e570 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala index c04d3b2db29..72d7b5a6f32 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRow.scala @@ -14,9 +14,6 @@ * limitations under the License. */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRowBase.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRowBase.scala similarity index 99% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRowBase.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRowBase.scala index e5e0bbd3dc6..fb39bca048a 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRowBase.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/CudfUnsafeRowBase.scala @@ -14,9 +14,6 @@ * limitations under the License. */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DateTimeUtilsShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DateTimeUtilsShims.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DateTimeUtilsShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DateTimeUtilsShims.scala index 327b94a5d27..a4e2e09b0f4 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DateTimeUtilsShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DateTimeUtilsShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DecimalArithmeticOverrides.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DecimalArithmeticOverrides.scala similarity index 99% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DecimalArithmeticOverrides.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DecimalArithmeticOverrides.scala index 547d56747fa..a31f63baea2 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DecimalArithmeticOverrides.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DecimalArithmeticOverrides.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DecimalMultiply128.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DecimalMultiply128.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DecimalMultiply128.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DecimalMultiply128.scala index a237fc13144..f270b6b901f 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DecimalMultiply128.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DecimalMultiply128.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DeltaLakeUtils.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DeltaLakeUtils.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DeltaLakeUtils.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DeltaLakeUtils.scala index 6bdce1011fa..56c2907dd87 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DeltaLakeUtils.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DeltaLakeUtils.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DistributionUtil.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DistributionUtil.scala similarity index 93% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DistributionUtil.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DistributionUtil.scala index d23bfdf0a6f..54d01e7a2a3 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DistributionUtil.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/DistributionUtil.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/FileIndexOptionsShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/FileIndexOptionsShims.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/FileIndexOptionsShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/FileIndexOptionsShims.scala index 85a3a324c97..4c39738e4d0 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/FileIndexOptionsShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/FileIndexOptionsShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GetMapValueMeta.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GetMapValueMeta.scala similarity index 97% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GetMapValueMeta.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GetMapValueMeta.scala index a3a3dafdb5c..6a3150a95b4 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GetMapValueMeta.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GetMapValueMeta.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GetSequenceSize.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GetSequenceSize.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GetSequenceSize.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GetSequenceSize.scala index 8ad2839ab78..32ca03974bf 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GetSequenceSize.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GetSequenceSize.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GlobalLimitShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GlobalLimitShims.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GlobalLimitShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GlobalLimitShims.scala index 5043b0c3ce5..5320686e499 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GlobalLimitShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GlobalLimitShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuAggregateInPandasExecMeta.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuAggregateInPandasExecMeta.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuAggregateInPandasExecMeta.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuAggregateInPandasExecMeta.scala index 7bb97012966..2752c89463a 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuAggregateInPandasExecMeta.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuAggregateInPandasExecMeta.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuBroadcastJoinMeta.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuBroadcastJoinMeta.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuBroadcastJoinMeta.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuBroadcastJoinMeta.scala index b9e04808deb..fd527976f8f 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuBroadcastJoinMeta.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuBroadcastJoinMeta.scala @@ -14,9 +14,6 @@ * limitations under the License. */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuCastShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuCastShims.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuCastShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuCastShims.scala index dfe8a5a528f..9c28b40437a 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuCastShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuCastShims.scala @@ -16,9 +16,6 @@ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuFileFormatDataWriterShim.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuFileFormatDataWriterShim.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuFileFormatDataWriterShim.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuFileFormatDataWriterShim.scala index 3f4829cc9d2..fce7dcbd356 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuFileFormatDataWriterShim.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuFileFormatDataWriterShim.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuHashPartitioning.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuHashPartitioning.scala similarity index 94% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuHashPartitioning.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuHashPartitioning.scala index 712a7414216..f906bd28acc 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuHashPartitioning.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuHashPartitioning.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtils.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtils.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtils.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtils.scala index 04aef3defae..dee791ef29b 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtils.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuIntervalUtils.scala @@ -14,9 +14,6 @@ * limitations under the License. */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala similarity index 99% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala index ff9ca51b30a..825ece4c33d 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuRangePartitioning.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuRangePartitioning.scala similarity index 97% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuRangePartitioning.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuRangePartitioning.scala index fef46c61b73..adb9619340f 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuRangePartitioning.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuRangePartitioning.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * Copyright (c) 2020-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuTypeShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuTypeShims.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuTypeShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuTypeShims.scala index 2a55034b698..252309f254f 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuTypeShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuTypeShims.scala @@ -14,9 +14,6 @@ * limitations under the License. */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuWindowInPandasExec.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuWindowInPandasExec.scala similarity index 97% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuWindowInPandasExec.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuWindowInPandasExec.scala index 5ec195bd554..eb65ea3fbef 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuWindowInPandasExec.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/GpuWindowInPandasExec.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/InSubqueryShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/InSubqueryShims.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/InSubqueryShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/InSubqueryShims.scala index 504667b0fb4..91f9e492ec6 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/InSubqueryShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/InSubqueryShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala index 781cac2c580..5c176c48ce2 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala index bdb732e54b2..1708cf194bb 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcProtoWriterShim.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcProtoWriterShim.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcProtoWriterShim.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcProtoWriterShim.scala index b0c2b505e40..624f9a7664a 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcProtoWriterShim.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcProtoWriterShim.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcReadingShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcReadingShims.scala similarity index 97% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcReadingShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcReadingShims.scala index fc3be7f6609..9277501e174 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcReadingShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/OrcReadingShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetFieldIdShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetFieldIdShims.scala similarity index 93% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetFieldIdShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetFieldIdShims.scala index 340dce31388..248fd3d30a1 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetFieldIdShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetFieldIdShims.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetLegacyNanoAsLongShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetLegacyNanoAsLongShims.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetLegacyNanoAsLongShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetLegacyNanoAsLongShims.scala index 0880cc8d24c..bead31833eb 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetLegacyNanoAsLongShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetLegacyNanoAsLongShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetStringPredShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetStringPredShims.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetStringPredShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetStringPredShims.scala index c1a4a0d4c71..677b0f9fe63 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetStringPredShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetStringPredShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetTimestampNTZShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetTimestampNTZShims.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetTimestampNTZShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetTimestampNTZShims.scala index 026dfe45d15..58ea0e732c2 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetTimestampNTZShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ParquetTimestampNTZShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala similarity index 93% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala index 4a12f723a79..c8f9024e352 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/PartitionedFileUtilsShim.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/PythonUDFShim.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/PythonUDFShim.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/PythonUDFShim.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/PythonUDFShim.scala index cd1ebe4b59c..8bed67b6e7b 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/PythonUDFShim.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/PythonUDFShim.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala similarity index 97% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala index de433d5f270..26ec91dd6f0 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RaiseErrorShim.scala @@ -14,9 +14,6 @@ * limitations under the License. */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RapidsFileSourceMetaUtils.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RapidsFileSourceMetaUtils.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RapidsFileSourceMetaUtils.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RapidsFileSourceMetaUtils.scala index 06a8016f6b2..fa91d37aa23 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/RapidsFileSourceMetaUtils.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/RapidsFileSourceMetaUtils.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimFilePartitionReaderFactory.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimFilePartitionReaderFactory.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimFilePartitionReaderFactory.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimFilePartitionReaderFactory.scala index c0791021c3f..701b94671a9 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimFilePartitionReaderFactory.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimFilePartitionReaderFactory.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimLeafExecNode.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimLeafExecNode.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimLeafExecNode.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimLeafExecNode.scala index 5e569c484cd..7196f3acc62 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimLeafExecNode.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/ShimLeafExecNode.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/Spark31Xuntil33XShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/Spark31Xuntil33XShims.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/Spark31Xuntil33XShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/Spark31Xuntil33XShims.scala index 95ca4019b84..9f788c81dd7 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/Spark31Xuntil33XShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/Spark31Xuntil33XShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/TypeUtilsShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/TypeUtilsShims.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/TypeUtilsShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/TypeUtilsShims.scala index 4a678371f35..32aecfaca02 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/TypeUtilsShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/TypeUtilsShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/extractValueShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/extractValueShims.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/extractValueShims.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/extractValueShims.scala index 1f2514a10d4..8c47d714b49 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/extractValueShims.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/extractValueShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala similarity index 99% rename from sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala rename to sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala index bfad2278344..70942001cae 100644 --- a/sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleIterator.scala @@ -16,9 +16,6 @@ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala index 2dcad0d4226..036b5838daf 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala similarity index 91% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala index e740930305b..68f148c9d27 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/catalyst/json/GpuJsonUtils.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/catalyst/json/GpuJsonUtils.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/catalyst/json/GpuJsonUtils.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/catalyst/json/GpuJsonUtils.scala index ce5ecff513c..cd523d44a2c 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/catalyst/json/GpuJsonUtils.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/catalyst/json/GpuJsonUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/execution/datasources/parquet/ShimCurrentBatchIterator.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/execution/datasources/parquet/ShimCurrentBatchIterator.scala similarity index 99% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/execution/datasources/parquet/ShimCurrentBatchIterator.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/execution/datasources/parquet/ShimCurrentBatchIterator.scala index bf43fbf0454..6e0a0acbc0c 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/execution/datasources/parquet/ShimCurrentBatchIterator.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/execution/datasources/parquet/ShimCurrentBatchIterator.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/execution/rapids/shims/FilePartitionShims.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/execution/rapids/shims/FilePartitionShims.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/execution/rapids/shims/FilePartitionShims.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/execution/rapids/shims/FilePartitionShims.scala index aabb2dd5e36..8b6e43fe60b 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/execution/rapids/shims/FilePartitionShims.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/execution/rapids/shims/FilePartitionShims.scala @@ -16,9 +16,6 @@ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala index 1e1ac57aa60..5d1e20ae18b 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/FileSinkDescShim.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/FileSinkDescShim.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/FileSinkDescShim.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/FileSinkDescShim.scala index 9bceecac524..be7d74bb82c 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/FileSinkDescShim.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/FileSinkDescShim.scala @@ -16,9 +16,6 @@ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala similarity index 99% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala index 216235accf0..3b33ea5c9ac 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/GpuCreateHiveTableAsSelectCommand.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala similarity index 99% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala index 08be08b9cd6..9dd038b8874 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuRowBasedHiveGenericUDFShim.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/GpuRowBasedHiveGenericUDFShim.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuRowBasedHiveGenericUDFShim.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/GpuRowBasedHiveGenericUDFShim.scala index fe4c37c9ef9..3021ec7e91d 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuRowBasedHiveGenericUDFShim.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/GpuRowBasedHiveGenericUDFShim.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/HiveInspectorsShim.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/HiveInspectorsShim.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/HiveInspectorsShim.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/HiveInspectorsShim.scala index 84d653f1d05..40de4f6d329 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/HiveInspectorsShim.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/HiveInspectorsShim.scala @@ -16,9 +16,6 @@ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/HiveProviderCmdShims.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/HiveProviderCmdShims.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/HiveProviderCmdShims.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/HiveProviderCmdShims.scala index c991b00bec4..930ab0e858c 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/HiveProviderCmdShims.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/hive/rapids/shims/HiveProviderCmdShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuDataSource.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/GpuDataSource.scala similarity index 99% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuDataSource.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/GpuDataSource.scala index 0b9fae05b6a..0bbdc614967 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuDataSource.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/GpuDataSource.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala similarity index 99% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala index 18446eca8a1..549ac204454 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/RapidsCachingReader.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/RapidsCachingReader.scala similarity index 99% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/RapidsCachingReader.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/RapidsCachingReader.scala index d68305655cd..3334187bb16 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/RapidsCachingReader.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/RapidsCachingReader.scala @@ -16,9 +16,6 @@ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/aggregate/aggregateFunctions.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/aggregate/aggregateFunctions.scala similarity index 97% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/aggregate/aggregateFunctions.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/aggregate/aggregateFunctions.scala index 767665b97c4..344a4898b24 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/aggregate/aggregateFunctions.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/aggregate/aggregateFunctions.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/arithmetic.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/arithmetic.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/arithmetic.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/arithmetic.scala index f38af394591..73992d1ecd0 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/arithmetic.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/arithmetic.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala index efc8fc19147..c6d82e73be1 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastHashJoinExec.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastNestedLoopJoinExec.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastNestedLoopJoinExec.scala similarity index 99% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastNestedLoopJoinExec.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastNestedLoopJoinExec.scala index 059a7e1426b..901dc83d576 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastNestedLoopJoinExec.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/GpuBroadcastNestedLoopJoinExec.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuShuffleMeta.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/GpuShuffleMeta.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuShuffleMeta.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/GpuShuffleMeta.scala index 84f3313452f..4cd1aebdd20 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuShuffleMeta.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/GpuShuffleMeta.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala index 9bcfa33ab87..d72cea9e168 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/GpuSubqueryBroadcastMeta.scala @@ -14,9 +14,6 @@ * limitations under the License. */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala index 977c755712a..32818f52ee0 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuBasePythonRunner.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuBasePythonRunner.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuBasePythonRunner.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuBasePythonRunner.scala index e7245db64e4..971b0eedd0e 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuBasePythonRunner.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuBasePythonRunner.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala similarity index 98% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala index 68112676a2b..4835b3d4b86 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala similarity index 97% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala index 9df93a9d11b..a5a8af5b8a0 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala index aacf972e7e0..8ff6cdcbde6 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/WritePythonUDFUtils.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/ArrowUtilsShim.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/ArrowUtilsShim.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/ArrowUtilsShim.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/ArrowUtilsShim.scala index 63f132bd2f8..8496061dc86 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/ArrowUtilsShim.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/ArrowUtilsShim.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/DataTypeUtilsShim.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/DataTypeUtilsShim.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/DataTypeUtilsShim.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/DataTypeUtilsShim.scala index 41aa0807646..28839326183 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/DataTypeUtilsShim.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/DataTypeUtilsShim.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala similarity index 97% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala index 24d33cc7155..6ae300cb108 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/GpuAscii.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala similarity index 99% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala index f5f65dce10a..c365ebe5c5d 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/GpuCreateDataSourceTableAsSelectCommandShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuMapInPandasExecMeta.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/GpuMapInPandasExecMeta.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuMapInPandasExecMeta.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/GpuMapInPandasExecMeta.scala index 05096f6a41e..9482b8f594f 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/GpuMapInPandasExecMeta.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/GpuMapInPandasExecMeta.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsHadoopWriterUtils.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsHadoopWriterUtils.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsHadoopWriterUtils.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsHadoopWriterUtils.scala index 20a63c70bb4..5c58ef485f6 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/RapidsHadoopWriterUtils.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/RapidsHadoopWriterUtils.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/SchemaUtilsShims.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/SchemaUtilsShims.scala similarity index 97% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/SchemaUtilsShims.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/SchemaUtilsShims.scala index fa806582664..3052d06e8da 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/SchemaUtilsShims.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/SchemaUtilsShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala similarity index 96% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala index d67d078c107..61b1b2d2267 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/SparkUpgradeExceptionShims.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/misc.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/misc.scala similarity index 95% rename from sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/misc.scala rename to sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/misc.scala index 1ab58ddcbb6..f4fd97b00e2 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/misc.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/shims/misc.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,6 @@ * limitations under the License. */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/sql-plugin/src/test/spark311/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimsSuite.scala b/sql-plugin/src/test/spark311/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimsSuite.scala deleted file mode 100644 index 34c598e83b3..00000000000 --- a/sql-plugin/src/test/spark311/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimsSuite.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "311"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark311 - -import com.nvidia.spark.rapids._ -import org.scalatest.funsuite.AnyFunSuite - -class SparkShimsSuite extends AnyFunSuite with FQSuiteName { - test("spark shims version") { - assert(ShimLoader.getShimVersion === SparkShimVersion(3, 1, 1)) - } - - test("shuffle manager class") { - assert(ShimLoader.getRapidsShuffleManagerClass === - classOf[com.nvidia.spark.rapids.spark311.RapidsShuffleManager].getCanonicalName) - } -} diff --git a/sql-plugin/src/test/spark312/scala/com/nvidia/spark/rapids/shims/spark312/SparkShimsSuite.scala b/sql-plugin/src/test/spark312/scala/com/nvidia/spark/rapids/shims/spark312/SparkShimsSuite.scala deleted file mode 100644 index 075394bb2b1..00000000000 --- a/sql-plugin/src/test/spark312/scala/com/nvidia/spark/rapids/shims/spark312/SparkShimsSuite.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "312"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark312 - -import com.nvidia.spark.rapids._ -import org.scalatest.funsuite.AnyFunSuite - -class SparkShimsSuite extends AnyFunSuite with FQSuiteName { - test("spark shims version") { - assert(ShimLoader.getShimVersion === SparkShimVersion(3, 1, 2)) - } - - test("shuffle manager class") { - assert(ShimLoader.getRapidsShuffleManagerClass === - classOf[com.nvidia.spark.rapids.spark312.RapidsShuffleManager].getCanonicalName) - } -} diff --git a/sql-plugin/src/test/spark313/scala/com/nvidia/spark/rapids/shims/spark313/SparkShimsSuite.scala b/sql-plugin/src/test/spark313/scala/com/nvidia/spark/rapids/shims/spark313/SparkShimsSuite.scala deleted file mode 100644 index fd663661e8c..00000000000 --- a/sql-plugin/src/test/spark313/scala/com/nvidia/spark/rapids/shims/spark313/SparkShimsSuite.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/*** spark-rapids-shim-json-lines -{"spark": "313"} -spark-rapids-shim-json-lines ***/ -package com.nvidia.spark.rapids.shims.spark313 - -import com.nvidia.spark.rapids._ -import org.scalatest.funsuite.AnyFunSuite - -class SparkShimsSuite extends AnyFunSuite with FQSuiteName { - test("spark shims version") { - assert(ShimLoader.getShimVersion === SparkShimVersion(3, 1, 3)) - } - - test("shuffle manager class") { - assert(ShimLoader.getRapidsShuffleManagerClass === - classOf[com.nvidia.spark.rapids.spark313.RapidsShuffleManager].getCanonicalName) - } -} diff --git a/tests/README.md b/tests/README.md index c854a2e6625..d004e07d009 100644 --- a/tests/README.md +++ b/tests/README.md @@ -44,9 +44,9 @@ and the the #### Running Unit Tests Against Specific Apache Spark Versions You can run the unit tests against different versions of Spark using the different profiles. The -default version runs against Spark 3.1.1, to run against a specific version use a buildver property: +default version runs against Spark 3.2.0, to run against a specific version use a buildver property: -- `-Dbuildver=311` (Spark 3.1.1) +- `-Dbuildver=320` (Spark 3.2.0) - `-Dbuildver=350` (Spark 3.5.0) etc diff --git a/tests/src/test/scala/com/nvidia/spark/rapids/timezone/TimeZonePerfSuite.scala b/tests/src/test/scala/com/nvidia/spark/rapids/timezone/TimeZonePerfSuite.scala index d3388c68931..a9618a448cf 100644 --- a/tests/src/test/scala/com/nvidia/spark/rapids/timezone/TimeZonePerfSuite.scala +++ b/tests/src/test/scala/com/nvidia/spark/rapids/timezone/TimeZonePerfSuite.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.types._ * Usage: * * argLine="-DTZs=Asia/Shanghai,Japan -DenableTimeZonePerf=true" \ - * mvn test -Dbuildver=311 -DwildcardSuites=com.nvidia.spark.rapids.timezone.TimeZonePerfSuite + * mvn test -Dbuildver=320 -DwildcardSuites=com.nvidia.spark.rapids.timezone.TimeZonePerfSuite * Note: * Generate a Parquet file with 6 columns: * - c_ts: timestamp column diff --git a/tests/src/test/spark311/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleTestHelper.scala b/tests/src/test/spark320/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleTestHelper.scala similarity index 99% rename from tests/src/test/spark311/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleTestHelper.scala rename to tests/src/test/spark320/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleTestHelper.scala index 73d081137cd..ab303d8098e 100644 --- a/tests/src/test/spark311/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleTestHelper.scala +++ b/tests/src/test/spark320/scala/com/nvidia/spark/rapids/shuffle/RapidsShuffleTestHelper.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "320"} {"spark": "321"} {"spark": "321cdh"} diff --git a/tests/src/test/spark311/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala b/tests/src/test/spark321cdh/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala similarity index 97% rename from tests/src/test/spark311/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala rename to tests/src/test/spark321cdh/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala index 6f6f49f113b..56e3a114fda 100644 --- a/tests/src/test/spark311/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala +++ b/tests/src/test/spark321cdh/scala/com/nvidia/spark/rapids/shims/OrcStatisticShim.scala @@ -15,9 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "311"} -{"spark": "312"} -{"spark": "313"} {"spark": "321cdh"} {"spark": "330cdh"} {"spark": "332cdh"} diff --git a/tools/generated_files/operatorsScore.csv b/tools/generated_files/operatorsScore.csv index c5c2080694c..0c7295f95ac 100644 --- a/tools/generated_files/operatorsScore.csv +++ b/tools/generated_files/operatorsScore.csv @@ -14,14 +14,18 @@ SortExec,8.0 SubqueryBroadcastExec,3.0 TakeOrderedAndProjectExec,3.0 UnionExec,3.0 -CustomShuffleReaderExec,3.0 +AQEShuffleReadExec,3.0 HashAggregateExec,4.5 ObjectHashAggregateExec,3.0 SortAggregateExec,3.0 InMemoryTableScanExec,3.0 DataWritingCommandExec,3.0 ExecutedCommandExec,3.0 +AppendDataExecV1,3.0 +AtomicCreateTableAsSelectExec,3.0 +AtomicReplaceTableAsSelectExec,3.0 BatchScanExec,3.0 +OverwriteByExpressionExecV1,3.0 BroadcastExchangeExec,3.0 ShuffleExchangeExec,4.2 BroadcastHashJoinExec,5.1 diff --git a/tools/generated_files/supportedDataSource.csv b/tools/generated_files/supportedDataSource.csv index a69379bff51..2573406ec3b 100644 --- a/tools/generated_files/supportedDataSource.csv +++ b/tools/generated_files/supportedDataSource.csv @@ -1,13 +1,13 @@ -Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT -Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO -CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA -Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS -Delta,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS -HiveText,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS -JSON,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO -ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS -ORC,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS -Parquet,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH +Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO +CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NA,NA +Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA +Delta,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS +HiveText,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA +JSON,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO +ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA +ORC,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,NA,NA +Parquet,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/tools/generated_files/supportedExecs.csv b/tools/generated_files/supportedExecs.csv index f5a3fe7c4b5..4eee8a7ad43 100644 --- a/tools/generated_files/supportedExecs.csv +++ b/tools/generated_files/supportedExecs.csv @@ -1,50 +1,54 @@ -Exec,Supported,Notes,Params,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT -CoalesceExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -CollectLimitExec,NS,This is disabled by default because Collect Limit replacement can be slower on the GPU; if huge number of rows in a batch it could help by limiting the number of rows transferred from GPU to CPU,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -ExpandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -FileSourceScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -FilterExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -GenerateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -GlobalLimitExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -LocalLimitExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -ProjectExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -RangeExec,S,None,Input/Output,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -SampleExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -SortExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -SubqueryBroadcastExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -TakeOrderedAndProjectExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -UnionExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -CustomShuffleReaderExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS -ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS -SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS -DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS -ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -BatchScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS -BroadcastExchangeExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -ShuffleExchangeExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -BroadcastHashJoinExec,S,None,leftKeys,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -BroadcastHashJoinExec,S,None,rightKeys,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -BroadcastHashJoinExec,S,None,condition,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BroadcastHashJoinExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -BroadcastNestedLoopJoinExec,S,None,condition(A non-inner join only is supported if the condition expression can be converted to a GPU AST expression),S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BroadcastNestedLoopJoinExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -CartesianProductExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -ShuffledHashJoinExec,S,None,leftKeys,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -ShuffledHashJoinExec,S,None,rightKeys,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -ShuffledHashJoinExec,S,None,condition,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShuffledHashJoinExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -SortMergeJoinExec,S,None,leftKeys,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -SortMergeJoinExec,S,None,rightKeys,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -SortMergeJoinExec,S,None,condition,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -SortMergeJoinExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -AggregateInPandasExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS -ArrowEvalPythonExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS -FlatMapCoGroupsInPandasExec,NS,This is disabled by default because Performance is not ideal with many small groups,Input/Output,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS -FlatMapGroupsInPandasExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS -MapInPandasExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS -WindowInPandasExec,NS,This is disabled by default because it only supports row based frame for now,Input/Output,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,NS,NS -WindowExec,S,None,partitionSpec,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,PS,NS -WindowExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -HiveTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS +Exec,Supported,Notes,Params,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH +CoalesceExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +CollectLimitExec,NS,This is disabled by default because Collect Limit replacement can be slower on the GPU; if huge number of rows in a batch it could help by limiting the number of rows transferred from GPU to CPU,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +ExpandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +FileSourceScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +FilterExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +GenerateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +GlobalLimitExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +LocalLimitExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +ProjectExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +RangeExec,S,None,Input/Output,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +SampleExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +SortExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +SubqueryBroadcastExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +TakeOrderedAndProjectExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +UnionExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS +ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS +SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,NS,NS +DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,NS,NS +ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS +AtomicCreateTableAsSelectExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS +AtomicReplaceTableAsSelectExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS +BatchScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS +OverwriteByExpressionExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS +BroadcastExchangeExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +ShuffleExchangeExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +BroadcastHashJoinExec,S,None,leftKeys,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NS,NS +BroadcastHashJoinExec,S,None,rightKeys,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NS,NS +BroadcastHashJoinExec,S,None,condition,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BroadcastHashJoinExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +BroadcastNestedLoopJoinExec,S,None,condition(A non-inner join only is supported if the condition expression can be converted to a GPU AST expression),S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BroadcastNestedLoopJoinExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +CartesianProductExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +ShuffledHashJoinExec,S,None,leftKeys,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NS,NS +ShuffledHashJoinExec,S,None,rightKeys,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NS,NS +ShuffledHashJoinExec,S,None,condition,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShuffledHashJoinExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +SortMergeJoinExec,S,None,leftKeys,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NS,NS +SortMergeJoinExec,S,None,rightKeys,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NS,NS +SortMergeJoinExec,S,None,condition,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +SortMergeJoinExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +AggregateInPandasExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS +ArrowEvalPythonExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS +FlatMapCoGroupsInPandasExec,NS,This is disabled by default because Performance is not ideal with many small groups,Input/Output,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS +FlatMapGroupsInPandasExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS +MapInPandasExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS +WindowInPandasExec,NS,This is disabled by default because it only supports row based frame for now,Input/Output,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,NS,NS,NS,NS +WindowExec,S,None,partitionSpec,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,PS,NS,NS,NS +WindowExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +HiveTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS diff --git a/tools/generated_files/supportedExprs.csv b/tools/generated_files/supportedExprs.csv index 88f6ee0d6d3..f9e47f10110 100644 --- a/tools/generated_files/supportedExprs.csv +++ b/tools/generated_files/supportedExprs.csv @@ -1,742 +1,742 @@ -Expression,Supported,SQL Func,Notes,Context,Params,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT -Abs,S,`abs`,None,project,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Abs,S,`abs`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Abs,S,`abs`,None,AST,input,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA -Abs,S,`abs`,None,AST,result,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA -Acos,S,`acos`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Acos,S,`acos`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Acos,S,`acos`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Acos,S,`acos`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Acosh,S,`acosh`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Acosh,S,`acosh`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Acosh,S,`acosh`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Acosh,S,`acosh`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Add,S,`+`,None,project,lhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA -Add,S,`+`,None,project,rhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA -Add,S,`+`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA -Add,S,`+`,None,AST,lhs,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA -Add,S,`+`,None,AST,rhs,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA -Add,S,`+`,None,AST,result,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA -Alias,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -Alias,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -Alias,S, ,None,AST,input,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS -Alias,S, ,None,AST,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS -And,S,`and`,None,project,lhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -And,S,`and`,None,project,rhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -And,S,`and`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -And,S,`and`,None,AST,lhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -And,S,`and`,None,AST,rhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -And,S,`and`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ArrayContains,S,`array_contains`,None,project,array,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayContains,S,`array_contains`,None,project,key,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS -ArrayContains,S,`array_contains`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ArrayExcept,S,`array_except`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayExcept,S,`array_except`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayExcept,S,`array_except`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayExists,S,`exists`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayExists,S,`exists`,None,project,function,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ArrayExists,S,`exists`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ArrayFilter,S,`filter`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayFilter,S,`filter`,None,project,function,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ArrayFilter,S,`filter`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayIntersect,S,`array_intersect`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayIntersect,S,`array_intersect`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayIntersect,S,`array_intersect`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayMax,S,`array_max`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayMax,S,`array_max`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS -ArrayMin,S,`array_min`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayMin,S,`array_min`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS -ArrayRemove,S,`array_remove`,None,project,array,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,PS,NS,NS,NS -ArrayRemove,S,`array_remove`,None,project,element,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -ArrayRemove,S,`array_remove`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayRepeat,S,`array_repeat`,None,project,left,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -ArrayRepeat,S,`array_repeat`,None,project,right,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ArrayRepeat,S,`array_repeat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayTransform,S,`transform`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayTransform,S,`transform`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -ArrayTransform,S,`transform`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayUnion,S,`array_union`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayUnion,S,`array_union`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArrayUnion,S,`array_union`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ArraysZip,S,`arrays_zip`,None,project,children,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ArraysZip,S,`arrays_zip`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -Ascii,NS,`ascii`,This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters after Spark 3.2.3; 3.3.1 and 3.4.0. Otherwise the results will not match the CPU.,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -Ascii,NS,`ascii`,This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters after Spark 3.2.3; 3.3.1 and 3.4.0. Otherwise the results will not match the CPU.,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Asin,S,`asin`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Asin,S,`asin`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Asin,S,`asin`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Asin,S,`asin`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Asinh,S,`asinh`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Asinh,S,`asinh`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Asinh,S,`asinh`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Asinh,S,`asinh`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -AtLeastNNonNulls,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -AtLeastNNonNulls,S, ,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Atan,S,`atan`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Atan,S,`atan`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Atan,S,`atan`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Atan,S,`atan`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Atanh,S,`atanh`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Atanh,S,`atanh`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Atanh,S,`atanh`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Atanh,S,`atanh`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -AttributeReference,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -AttributeReference,S, ,None,AST,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS -BRound,S,`bround`,None,project,value,NA,S,S,S,S,PS,PS,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -BRound,S,`bround`,None,project,scale,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BRound,S,`bround`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -BitLength,S,`bit_length`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA -BitLength,S,`bit_length`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseAnd,S,`&`,None,project,lhs,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseAnd,S,`&`,None,project,rhs,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseAnd,S,`&`,None,project,result,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseAnd,S,`&`,None,AST,lhs,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseAnd,S,`&`,None,AST,rhs,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseAnd,S,`&`,None,AST,result,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseNot,S,`~`,None,project,input,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseNot,S,`~`,None,project,result,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseNot,S,`~`,None,AST,input,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseNot,S,`~`,None,AST,result,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseOr,S,`\|`,None,project,lhs,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseOr,S,`\|`,None,project,rhs,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseOr,S,`\|`,None,project,result,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseOr,S,`\|`,None,AST,lhs,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseOr,S,`\|`,None,AST,rhs,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseOr,S,`\|`,None,AST,result,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseXor,S,`^`,None,project,lhs,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseXor,S,`^`,None,project,rhs,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseXor,S,`^`,None,project,result,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseXor,S,`^`,None,AST,lhs,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseXor,S,`^`,None,AST,rhs,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BitwiseXor,S,`^`,None,AST,result,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -BoundReference,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -BoundReference,S, ,None,AST,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS -CaseWhen,S,`when`,None,project,predicate,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -CaseWhen,S,`when`,None,project,value,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -CaseWhen,S,`when`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -Cbrt,S,`cbrt`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cbrt,S,`cbrt`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cbrt,S,`cbrt`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cbrt,S,`cbrt`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Ceil,S,`ceil`; `ceiling`,None,project,input,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Ceil,S,`ceil`; `ceiling`,None,project,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -CheckOverflow,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -CheckOverflow,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Coalesce,S,`coalesce`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -Coalesce,S,`coalesce`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -Concat,S,`concat`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,PS,NA,NA,NA -Concat,S,`concat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,PS,NA,NA,NA -ConcatWs,S,`concat_ws`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,S,NA,NA,NA -ConcatWs,S,`concat_ws`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -Contains,S, ,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -Contains,S, ,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -Contains,S, ,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Conv,NS,`conv`,This is disabled by default because GPU implementation is incomplete. We currently only support from/to_base values of 10 and 16. We fall back on CPU if the signed conversion is signalled via a negative to_base. GPU implementation does not check for an 64-bit signed/unsigned int overflow when performing the conversion to return `FFFFFFFFFFFFFFFF` or `18446744073709551615` or to throw an error in the ANSI mode. It is safe to enable if the overflow is not possible or detected externally. For instance decimal strings not longer than 18 characters / hexadecimal strings not longer than 15 characters disregarding the sign cannot cause an overflow. ,project,num,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -Conv,NS,`conv`,This is disabled by default because GPU implementation is incomplete. We currently only support from/to_base values of 10 and 16. We fall back on CPU if the signed conversion is signalled via a negative to_base. GPU implementation does not check for an 64-bit signed/unsigned int overflow when performing the conversion to return `FFFFFFFFFFFFFFFF` or `18446744073709551615` or to throw an error in the ANSI mode. It is safe to enable if the overflow is not possible or detected externally. For instance decimal strings not longer than 18 characters / hexadecimal strings not longer than 15 characters disregarding the sign cannot cause an overflow. ,project,from_base,NA,PS,PS,PS,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Conv,NS,`conv`,This is disabled by default because GPU implementation is incomplete. We currently only support from/to_base values of 10 and 16. We fall back on CPU if the signed conversion is signalled via a negative to_base. GPU implementation does not check for an 64-bit signed/unsigned int overflow when performing the conversion to return `FFFFFFFFFFFFFFFF` or `18446744073709551615` or to throw an error in the ANSI mode. It is safe to enable if the overflow is not possible or detected externally. For instance decimal strings not longer than 18 characters / hexadecimal strings not longer than 15 characters disregarding the sign cannot cause an overflow. ,project,to_base,NA,PS,PS,PS,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Conv,NS,`conv`,This is disabled by default because GPU implementation is incomplete. We currently only support from/to_base values of 10 and 16. We fall back on CPU if the signed conversion is signalled via a negative to_base. GPU implementation does not check for an 64-bit signed/unsigned int overflow when performing the conversion to return `FFFFFFFFFFFFFFFF` or `18446744073709551615` or to throw an error in the ANSI mode. It is safe to enable if the overflow is not possible or detected externally. For instance decimal strings not longer than 18 characters / hexadecimal strings not longer than 15 characters disregarding the sign cannot cause an overflow. ,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -Cos,S,`cos`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cos,S,`cos`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cos,S,`cos`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cos,S,`cos`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cosh,S,`cosh`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cosh,S,`cosh`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cosh,S,`cosh`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cosh,S,`cosh`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cot,S,`cot`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cot,S,`cot`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cot,S,`cot`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Cot,S,`cot`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -CreateArray,S,`array`,None,project,arg,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,NS,PS,NS -CreateArray,S,`array`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -CreateMap,S,`map`,None,project,key,S,S,S,S,S,S,S,S,PS,S,S,S,NA,NA,PS,NA,PS,NA -CreateMap,S,`map`,None,project,value,S,S,S,S,S,S,S,S,PS,S,S,S,NA,NA,PS,PS,PS,NA -CreateNamedStruct,S,`named_struct`; `struct`,None,project,name,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -CreateNamedStruct,S,`named_struct`; `struct`,None,project,value,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -CreateNamedStruct,S,`named_struct`; `struct`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA -CurrentRow$,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA -DateAdd,S,`date_add`,None,project,startDate,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateAdd,S,`date_add`,None,project,days,NA,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateAdd,S,`date_add`,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateAddInterval,S, ,None,project,start,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateAddInterval,S, ,None,project,interval,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA -DateAddInterval,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateDiff,S,`datediff`,None,project,lhs,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateDiff,S,`datediff`,None,project,rhs,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateDiff,S,`datediff`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateFormatClass,S,`date_format`,None,project,timestamp,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateFormatClass,S,`date_format`,None,project,strfmt,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -DateFormatClass,S,`date_format`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -DateSub,S,`date_sub`,None,project,startDate,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateSub,S,`date_sub`,None,project,days,NA,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateSub,S,`date_sub`,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DayOfMonth,S,`day`; `dayofmonth`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DayOfMonth,S,`day`; `dayofmonth`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DayOfWeek,S,`dayofweek`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DayOfWeek,S,`dayofweek`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DayOfYear,S,`dayofyear`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DayOfYear,S,`dayofyear`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DenseRank,S,`dense_rank`,None,window,ordering,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS -DenseRank,S,`dense_rank`,None,window,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Divide,S,`/`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Divide,S,`/`,None,project,rhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Divide,S,`/`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -DynamicPruningExpression,S, ,None,project,input,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DynamicPruningExpression,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -ElementAt,S,`element_at`,None,project,array/map,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,PS,NA,NA -ElementAt,S,`element_at`,None,project,index/key,PS,PS,PS,S,PS,PS,PS,PS,PS,PS,PS,NS,NS,NS,NS,NS,NS,NS -ElementAt,S,`element_at`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -EndsWith,S, ,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -EndsWith,S, ,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -EndsWith,S, ,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -EqualNullSafe,S,`<=>`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -EqualNullSafe,S,`<=>`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -EqualNullSafe,S,`<=>`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -EqualTo,S,`==`; `=`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -EqualTo,S,`==`; `=`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -EqualTo,S,`==`; `=`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -EqualTo,S,`==`; `=`,None,AST,lhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS -EqualTo,S,`==`; `=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS -EqualTo,S,`==`; `=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Exp,S,`exp`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Exp,S,`exp`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Exp,S,`exp`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Exp,S,`exp`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Explode,S,`explode_outer`; `explode`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,PS,NA,NA -Explode,S,`explode_outer`; `explode`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -Expm1,S,`expm1`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Expm1,S,`expm1`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Expm1,S,`expm1`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Expm1,S,`expm1`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Flatten,S,`flatten`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -Flatten,S,`flatten`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -Floor,S,`floor`,None,project,input,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Floor,S,`floor`,None,project,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -FormatNumber,S,`format_number`,None,project,x,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -FormatNumber,S,`format_number`,None,project,d,NA,NA,NA,PS,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA -FormatNumber,S,`format_number`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -FromUTCTimestamp,S,`from_utc_timestamp`,None,project,timestamp,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -FromUTCTimestamp,S,`from_utc_timestamp`,None,project,timezone,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -FromUTCTimestamp,S,`from_utc_timestamp`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -FromUnixTime,S,`from_unixtime`,None,project,sec,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -FromUnixTime,S,`from_unixtime`,None,project,format,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -FromUnixTime,S,`from_unixtime`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -GetArrayItem,S, ,None,project,array,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -GetArrayItem,S, ,None,project,ordinal,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -GetArrayItem,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -GetArrayStructFields,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -GetArrayStructFields,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -GetJsonObject,NS,`get_json_object`,This is disabled by default because Experimental feature that could be unstable or have performance issues.,project,json,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -GetJsonObject,NS,`get_json_object`,This is disabled by default because Experimental feature that could be unstable or have performance issues.,project,path,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -GetJsonObject,NS,`get_json_object`,This is disabled by default because Experimental feature that could be unstable or have performance issues.,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -GetMapValue,S, ,None,project,map,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA -GetMapValue,S, ,None,project,key,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS -GetMapValue,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -GetStructField,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA -GetStructField,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -GetTimestamp,S, ,None,project,timeExp,NA,NA,NA,NA,NA,NA,NA,S,PS,S,NA,NA,NA,NA,NA,NA,NA,NA -GetTimestamp,S, ,None,project,format,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -GetTimestamp,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -GreaterThan,S,`>`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -GreaterThan,S,`>`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -GreaterThan,S,`>`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -GreaterThan,S,`>`,None,AST,lhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS -GreaterThan,S,`>`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS -GreaterThan,S,`>`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -GreaterThanOrEqual,S,`>=`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -GreaterThanOrEqual,S,`>=`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -GreaterThanOrEqual,S,`>=`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -GreaterThanOrEqual,S,`>=`,None,AST,lhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS -GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS -GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS -Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS -HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS -HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Hypot,S,`hypot`,None,project,rhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Hypot,S,`hypot`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -If,S,`if`,None,project,predicate,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -If,S,`if`,None,project,trueValue,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -If,S,`if`,None,project,falseValue,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -If,S,`if`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -In,S,`in`,None,project,value,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS -In,S,`in`,None,project,list,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,NS,NS,NS,NS,NA,NS,NS -In,S,`in`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -InSet,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS -InSet,S, ,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -InitCap,S,`initcap`,This is not 100% compatible with the Spark version because the Unicode version used by cuDF and the JVM may differ; resulting in some corner-case characters not changing case correctly.,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -InitCap,S,`initcap`,This is not 100% compatible with the Spark version because the Unicode version used by cuDF and the JVM may differ; resulting in some corner-case characters not changing case correctly.,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -InputFileBlockLength,S,`input_file_block_length`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -InputFileBlockStart,S,`input_file_block_start`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -InputFileName,S,`input_file_name`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -IntegralDivide,S,`div`,None,project,lhs,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -IntegralDivide,S,`div`,None,project,rhs,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -IntegralDivide,S,`div`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -IsNaN,S,`isnan`,None,project,input,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -IsNaN,S,`isnan`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -IsNotNull,S,`isnotnull`,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -IsNotNull,S,`isnotnull`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -IsNull,S,`isnull`,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -IsNull,S,`isnull`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -JsonToStructs,NS,`from_json`,This is disabled by default because it is currently in beta and undergoes continuous enhancements. Please consult the [compatibility documentation](../compatibility.md#json-supporting-types) to determine whether you can enable this configuration for your use case,project,jsonStr,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -JsonToStructs,NS,`from_json`,This is disabled by default because it is currently in beta and undergoes continuous enhancements. Please consult the [compatibility documentation](../compatibility.md#json-supporting-types) to determine whether you can enable this configuration for your use case,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NS,PS,PS,NA -JsonTuple,NS,`json_tuple`,This is disabled by default because Experimental feature that could be unstable or have performance issues.,project,json,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -JsonTuple,NS,`json_tuple`,This is disabled by default because Experimental feature that could be unstable or have performance issues.,project,field,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -JsonTuple,NS,`json_tuple`,This is disabled by default because Experimental feature that could be unstable or have performance issues.,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA -KnownFloatingPointNormalized,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -KnownFloatingPointNormalized,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -KnownNotNull,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,NS,S,S,PS,PS,PS,NS -KnownNotNull,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,NS,S,S,PS,PS,PS,NS -Lag,S,`lag`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS -Lag,S,`lag`,None,window,offset,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Lag,S,`lag`,None,window,default,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS -Lag,S,`lag`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS -LambdaFunction,S, ,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -LambdaFunction,S, ,None,project,arguments,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -LambdaFunction,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -LastDay,S,`last_day`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -LastDay,S,`last_day`,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Lead,S,`lead`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS -Lead,S,`lead`,None,window,offset,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Lead,S,`lead`,None,window,default,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS -Lead,S,`lead`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS -Least,S,`least`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS -Least,S,`least`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS -Length,S,`char_length`; `character_length`; `length`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA -Length,S,`char_length`; `character_length`; `length`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -LessThan,S,`<`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -LessThan,S,`<`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -LessThan,S,`<`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -LessThan,S,`<`,None,AST,lhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS -LessThan,S,`<`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS -LessThan,S,`<`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -LessThanOrEqual,S,`<=`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -LessThanOrEqual,S,`<=`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS -LessThanOrEqual,S,`<=`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -LessThanOrEqual,S,`<=`,None,AST,lhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS -LessThanOrEqual,S,`<=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS -LessThanOrEqual,S,`<=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Like,S,`like`,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -Like,S,`like`,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -Like,S,`like`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Literal,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS -Literal,S, ,None,AST,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS -Log,S,`ln`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Log,S,`ln`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Log10,S,`log10`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Log10,S,`log10`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Log1p,S,`log1p`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Log1p,S,`log1p`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Log2,S,`log2`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Log2,S,`log2`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Logarithm,S,`log`,None,project,value,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Logarithm,S,`log`,None,project,base,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Logarithm,S,`log`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Lower,S,`lcase`; `lower`,This is not 100% compatible with the Spark version because the Unicode version used by cuDF and the JVM may differ; resulting in some corner-case characters not changing case correctly.,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -Lower,S,`lcase`; `lower`,This is not 100% compatible with the Spark version because the Unicode version used by cuDF and the JVM may differ; resulting in some corner-case characters not changing case correctly.,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -MakeDecimal,S, ,None,project,input,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -MakeDecimal,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA -MapConcat,S,`map_concat`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA -MapConcat,S,`map_concat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA -MapEntries,S,`map_entries`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA -MapEntries,S,`map_entries`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -MapFilter,S,`map_filter`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA -MapFilter,S,`map_filter`,None,project,function,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -MapFilter,S,`map_filter`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA -MapKeys,S,`map_keys`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA -MapKeys,S,`map_keys`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -MapValues,S,`map_values`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA -MapValues,S,`map_values`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -Md5,S,`md5`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA -Md5,S,`md5`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -MicrosToTimestamp,S,`timestamp_micros`,None,project,input,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -MicrosToTimestamp,S,`timestamp_micros`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -MillisToTimestamp,S,`timestamp_millis`,None,project,input,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -MillisToTimestamp,S,`timestamp_millis`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -Minute,S,`minute`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -Minute,S,`minute`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -MonotonicallyIncreasingID,S,`monotonically_increasing_id`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Month,S,`month`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Month,S,`month`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Multiply,S,`*`,None,project,lhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Multiply,S,`*`,None,project,rhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Multiply,S,`*`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Multiply,S,`*`,None,AST,lhs,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA -Multiply,S,`*`,None,AST,rhs,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA -Multiply,S,`*`,None,AST,result,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA -Murmur3Hash,S,`hash`,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS -Murmur3Hash,S,`hash`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -NaNvl,S,`nanvl`,None,project,lhs,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -NaNvl,S,`nanvl`,None,project,rhs,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -NaNvl,S,`nanvl`,None,project,result,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -NamedLambdaVariable,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -Not,S,`!`; `not`,None,project,input,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Not,S,`!`; `not`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Not,S,`!`; `not`,None,AST,input,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Not,S,`!`; `not`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -NthValue,S,`nth_value`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -NthValue,S,`nth_value`,None,window,offset,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -NthValue,S,`nth_value`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -OctetLength,S,`octet_length`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA -OctetLength,S,`octet_length`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Or,S,`or`,None,project,lhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Or,S,`or`,None,project,rhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Or,S,`or`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Or,S,`or`,None,AST,lhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Or,S,`or`,None,AST,rhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Or,S,`or`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ParseUrl,S,`parse_url`,None,project,url,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -ParseUrl,S,`parse_url`,None,project,partToExtract,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -ParseUrl,S,`parse_url`,None,project,key,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -ParseUrl,S,`parse_url`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -PercentRank,S,`percent_rank`,None,window,ordering,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS -PercentRank,S,`percent_rank`,None,window,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Pmod,S,`pmod`,None,project,lhs,NA,S,S,S,S,S,S,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA -Pmod,S,`pmod`,None,project,rhs,NA,S,S,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA -Pmod,S,`pmod`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA -PosExplode,S,`posexplode_outer`; `posexplode`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,PS,NA,NA -PosExplode,S,`posexplode_outer`; `posexplode`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -Pow,S,`pow`; `power`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Pow,S,`pow`; `power`,None,project,rhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Pow,S,`pow`; `power`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Pow,S,`pow`; `power`,None,AST,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Pow,S,`pow`; `power`,None,AST,rhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Pow,S,`pow`; `power`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -PreciseTimestampConversion,S, ,None,project,input,NA,NA,NA,NA,S,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -PreciseTimestampConversion,S, ,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -PromotePrecision,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -PromotePrecision,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -PythonUDF,S, ,None,aggregation,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS -PythonUDF,S, ,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA -PythonUDF,S, ,None,reduction,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS -PythonUDF,S, ,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA -PythonUDF,S, ,None,window,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS -PythonUDF,S, ,None,window,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA -PythonUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS -PythonUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA -Quarter,S,`quarter`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Quarter,S,`quarter`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -RLike,S,`rlike`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -RLike,S,`rlike`,None,project,regexp,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -RLike,S,`rlike`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -RaiseError,S,`raise_error`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -RaiseError,S,`raise_error`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA -Rand,S,`rand`; `random`,None,project,seed,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Rand,S,`rand`; `random`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Rank,S,`rank`,None,window,ordering,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS -Rank,S,`rank`,None,window,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -RegExpExtract,S,`regexp_extract`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -RegExpExtract,S,`regexp_extract`,None,project,regexp,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -RegExpExtract,S,`regexp_extract`,None,project,idx,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -RegExpExtract,S,`regexp_extract`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -RegExpExtractAll,S,`regexp_extract_all`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -RegExpExtractAll,S,`regexp_extract_all`,None,project,regexp,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -RegExpExtractAll,S,`regexp_extract_all`,None,project,idx,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -RegExpExtractAll,S,`regexp_extract_all`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA -RegExpReplace,S,`regexp_replace`,None,project,regex,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -RegExpReplace,S,`regexp_replace`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -RegExpReplace,S,`regexp_replace`,None,project,pos,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -RegExpReplace,S,`regexp_replace`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -RegExpReplace,S,`regexp_replace`,None,project,rep,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -Remainder,S,`%`; `mod`,None,project,lhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Remainder,S,`%`; `mod`,None,project,rhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Remainder,S,`%`; `mod`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -ReplicateRows,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS -ReplicateRows,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -Reverse,S,`reverse`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,PS,NA,NA,NA -Reverse,S,`reverse`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,PS,NA,NA,NA -Rint,S,`rint`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Rint,S,`rint`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Rint,S,`rint`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Rint,S,`rint`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Round,S,`round`,None,project,value,NA,S,S,S,S,PS,PS,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Round,S,`round`,None,project,scale,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Round,S,`round`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -RowNumber,S,`row_number`,None,window,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ScalaUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS -ScalaUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS -Second,S,`second`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -Second,S,`second`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -SecondsToTimestamp,S,`timestamp_seconds`,None,project,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -SecondsToTimestamp,S,`timestamp_seconds`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sequence,S,`sequence`,None,project,start,NA,S,S,S,S,NA,NA,NS,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sequence,S,`sequence`,None,project,stop,NA,S,S,S,S,NA,NA,NS,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sequence,S,`sequence`,None,project,step,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA -Sequence,S,`sequence`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -ShiftLeft,S,`shiftleft`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftLeft,S,`shiftleft`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftLeft,S,`shiftleft`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRight,S,`shiftright`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRight,S,`shiftright`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRight,S,`shiftright`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRightUnsigned,S,`shiftrightunsigned`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRightUnsigned,S,`shiftrightunsigned`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRightUnsigned,S,`shiftrightunsigned`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Signum,S,`sign`; `signum`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Signum,S,`sign`; `signum`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sin,S,`sin`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sin,S,`sin`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sin,S,`sin`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sin,S,`sin`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sinh,S,`sinh`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sinh,S,`sinh`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sinh,S,`sinh`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sinh,S,`sinh`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Size,S,`cardinality`; `size`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,PS,NA,NA -Size,S,`cardinality`; `size`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -SortArray,S,`sort_array`,None,project,array,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -SortArray,S,`sort_array`,None,project,ascendingOrder,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -SortArray,S,`sort_array`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -SortOrder,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS -SortOrder,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS -SparkPartitionID,S,`spark_partition_id`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -SpecifiedWindowFrame,S, ,None,project,lower,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,S,NA,NA,NA,NA -SpecifiedWindowFrame,S, ,None,project,upper,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,S,NA,NA,NA,NA -SpecifiedWindowFrame,S, ,None,project,result,NA,S,S,S,S,NS,NS,NA,NA,NA,NS,NA,NA,S,NA,NA,NA,NA -Sqrt,S,`sqrt`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sqrt,S,`sqrt`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sqrt,S,`sqrt`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sqrt,S,`sqrt`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Stack,S,`stack`,None,project,n,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Stack,S,`stack`,None,project,expr,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -Stack,S,`stack`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -StartsWith,S, ,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StartsWith,S, ,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -StartsWith,S, ,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringInstr,S,`instr`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringInstr,S,`instr`,None,project,substr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -StringInstr,S,`instr`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringLPad,S,`lpad`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringLPad,S,`lpad`,None,project,len,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringLPad,S,`lpad`,None,project,pad,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -StringLPad,S,`lpad`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringLocate,S,`locate`; `position`,None,project,substr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -StringLocate,S,`locate`; `position`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringLocate,S,`locate`; `position`,None,project,start,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringLocate,S,`locate`; `position`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringRPad,S,`rpad`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringRPad,S,`rpad`,None,project,len,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringRPad,S,`rpad`,None,project,pad,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -StringRPad,S,`rpad`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringRepeat,S,`repeat`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringRepeat,S,`repeat`,None,project,repeatTimes,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringRepeat,S,`repeat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringReplace,S,`replace`,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringReplace,S,`replace`,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -StringReplace,S,`replace`,None,project,replace,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -StringReplace,S,`replace`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringSplit,S,`split`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringSplit,S,`split`,None,project,regexp,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -StringSplit,S,`split`,None,project,limit,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringSplit,S,`split`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA -StringToMap,S,`str_to_map`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringToMap,S,`str_to_map`,None,project,pairDelim,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringToMap,S,`str_to_map`,None,project,keyValueDelim,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringToMap,S,`str_to_map`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA -StringTranslate,S,`translate`,This is not 100% compatible with the Spark version because the GPU implementation supports all unicode code points. In Spark versions < 3.2.0; translate() does not support unicode characters with code point >= U+10000 (See SPARK-34094),project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringTranslate,S,`translate`,This is not 100% compatible with the Spark version because the GPU implementation supports all unicode code points. In Spark versions < 3.2.0; translate() does not support unicode characters with code point >= U+10000 (See SPARK-34094),project,from,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -StringTranslate,S,`translate`,This is not 100% compatible with the Spark version because the GPU implementation supports all unicode code points. In Spark versions < 3.2.0; translate() does not support unicode characters with code point >= U+10000 (See SPARK-34094),project,to,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -StringTranslate,S,`translate`,This is not 100% compatible with the Spark version because the GPU implementation supports all unicode code points. In Spark versions < 3.2.0; translate() does not support unicode characters with code point >= U+10000 (See SPARK-34094),project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringTrim,S,`trim`,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringTrim,S,`trim`,None,project,trimStr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -StringTrim,S,`trim`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringTrimLeft,S,`ltrim`,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringTrimLeft,S,`ltrim`,None,project,trimStr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -StringTrimLeft,S,`ltrim`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringTrimRight,S,`rtrim`,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StringTrimRight,S,`rtrim`,None,project,trimStr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -StringTrimRight,S,`rtrim`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StructsToJson,NS,`to_json`,This is disabled by default because it is currently in beta and undergoes continuous enhancements. Please consult the [compatibility documentation](../compatibility.md#json-supporting-types) to determine whether you can enable this configuration for your use case,project,struct,S,S,S,S,S,S,S,S,PS,S,S,NA,NA,NA,PS,PS,PS,NA -StructsToJson,NS,`to_json`,This is disabled by default because it is currently in beta and undergoes continuous enhancements. Please consult the [compatibility documentation](../compatibility.md#json-supporting-types) to determine whether you can enable this configuration for your use case,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -Substring,S,`substr`; `substring`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA -Substring,S,`substr`; `substring`,None,project,pos,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Substring,S,`substr`; `substring`,None,project,len,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Substring,S,`substr`; `substring`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA -SubstringIndex,S,`substring_index`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -SubstringIndex,S,`substring_index`,None,project,delim,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -SubstringIndex,S,`substring_index`,None,project,count,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -SubstringIndex,S,`substring_index`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -Subtract,S,`-`,None,project,lhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA -Subtract,S,`-`,None,project,rhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA -Subtract,S,`-`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA -Subtract,S,`-`,None,AST,lhs,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA -Subtract,S,`-`,None,AST,rhs,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA -Subtract,S,`-`,None,AST,result,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA -Tan,S,`tan`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Tan,S,`tan`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Tan,S,`tan`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Tan,S,`tan`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Tanh,S,`tanh`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Tanh,S,`tanh`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Tanh,S,`tanh`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Tanh,S,`tanh`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -TimeAdd,S, ,None,project,start,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -TimeAdd,S, ,None,project,interval,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA -TimeAdd,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -ToDegrees,S,`degrees`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ToDegrees,S,`degrees`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ToRadians,S,`radians`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ToRadians,S,`radians`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ToUTCTimestamp,S,`to_utc_timestamp`,None,project,timestamp,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -ToUTCTimestamp,S,`to_utc_timestamp`,None,project,timezone,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -ToUTCTimestamp,S,`to_utc_timestamp`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA -ToUnixTimestamp,S,`to_unix_timestamp`,None,project,timeExp,NA,NA,NA,NA,NA,NA,NA,S,PS,S,NA,NA,NA,NA,NA,NA,NA,NA -ToUnixTimestamp,S,`to_unix_timestamp`,None,project,format,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -ToUnixTimestamp,S,`to_unix_timestamp`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -TransformKeys,S,`transform_keys`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA -TransformKeys,S,`transform_keys`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS -TransformKeys,S,`transform_keys`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA -TransformValues,S,`transform_values`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA -TransformValues,S,`transform_values`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS -TransformValues,S,`transform_values`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA -UnaryMinus,S,`negative`,None,project,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA -UnaryMinus,S,`negative`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA -UnaryMinus,S,`negative`,None,AST,input,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA -UnaryMinus,S,`negative`,None,AST,result,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA -UnaryPositive,S,`positive`,None,project,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA -UnaryPositive,S,`positive`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA -UnaryPositive,S,`positive`,None,AST,input,NA,S,S,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA -UnaryPositive,S,`positive`,None,AST,result,NA,S,S,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA -UnboundedFollowing$,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA -UnboundedPreceding$,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA -UnixTimestamp,S,`unix_timestamp`,None,project,timeExp,NA,NA,NA,NA,NA,NA,NA,S,PS,S,NA,NA,NA,NA,NA,NA,NA,NA -UnixTimestamp,S,`unix_timestamp`,None,project,format,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA -UnixTimestamp,S,`unix_timestamp`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -UnscaledValue,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA -UnscaledValue,S, ,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Upper,S,`ucase`; `upper`,This is not 100% compatible with the Spark version because the Unicode version used by cuDF and the JVM may differ; resulting in some corner-case characters not changing case correctly.,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -Upper,S,`ucase`; `upper`,This is not 100% compatible with the Spark version because the Unicode version used by cuDF and the JVM may differ; resulting in some corner-case characters not changing case correctly.,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -WeekDay,S,`weekday`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -WeekDay,S,`weekday`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -WindowExpression,S, ,None,window,windowFunction,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -WindowExpression,S, ,None,window,windowSpec,NA,S,S,S,S,NS,NS,NA,NA,NA,PS,NA,NA,S,NA,NA,NA,NA -WindowExpression,S, ,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -WindowSpecDefinition,S, ,None,project,partition,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,PS,NS -WindowSpecDefinition,S, ,None,project,value,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,PS,NS -WindowSpecDefinition,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,PS,NS -XxHash64,S,`xxhash64`,None,project,input,S,S,S,S,S,NS,NS,S,PS,S,S,S,NS,NS,NS,NS,NS,NS -XxHash64,S,`xxhash64`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Year,S,`year`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Year,S,`year`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -AggregateExpression,S, ,None,aggregation,aggFunc,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -AggregateExpression,S, ,None,aggregation,filter,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -AggregateExpression,S, ,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -AggregateExpression,S, ,None,reduction,aggFunc,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -AggregateExpression,S, ,None,reduction,filter,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -AggregateExpression,S, ,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -AggregateExpression,S, ,None,window,aggFunc,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -AggregateExpression,S, ,None,window,filter,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -AggregateExpression,S, ,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,aggregation,input,NA,S,S,S,S,S,S,NS,NS,NA,S,NA,NA,NA,NA,NA,NA,NA -ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,aggregation,percentage,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA -ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,aggregation,accuracy,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,aggregation,result,NA,S,S,S,S,S,S,NS,NS,NA,S,NA,NA,NA,PS,NA,NA,NA -ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,reduction,input,NA,S,S,S,S,S,S,NS,NS,NA,S,NA,NA,NA,NA,NA,NA,NA -ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,reduction,percentage,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA -ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,reduction,accuracy,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,reduction,result,NA,S,S,S,S,S,S,NS,NS,NA,S,NA,NA,NA,PS,NA,NA,NA -Average,S,`avg`; `mean`,None,aggregation,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Average,S,`avg`; `mean`,None,aggregation,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Average,S,`avg`; `mean`,None,reduction,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Average,S,`avg`; `mean`,None,reduction,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Average,S,`avg`; `mean`,None,window,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Average,S,`avg`; `mean`,None,window,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -CollectList,S,`collect_list`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -CollectList,S,`collect_list`,None,aggregation,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -CollectList,S,`collect_list`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -CollectList,S,`collect_list`,None,reduction,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -CollectList,S,`collect_list`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -CollectList,S,`collect_list`,None,window,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -CollectSet,S,`collect_set`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS -CollectSet,S,`collect_set`,None,aggregation,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -CollectSet,S,`collect_set`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS -CollectSet,S,`collect_set`,None,reduction,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -CollectSet,S,`collect_set`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS -CollectSet,S,`collect_set`,None,window,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA -Count,S,`count`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -Count,S,`count`,None,aggregation,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Count,S,`count`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -Count,S,`count`,None,reduction,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Count,S,`count`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S -Count,S,`count`,None,window,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -First,S,`first_value`; `first`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -First,S,`first_value`; `first`,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -First,S,`first_value`; `first`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -First,S,`first_value`; `first`,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -First,S,`first_value`; `first`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -First,S,`first_value`; `first`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -Last,S,`last_value`; `last`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -Last,S,`last_value`; `last`,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -Last,S,`last_value`; `last`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -Last,S,`last_value`; `last`,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -Last,S,`last_value`; `last`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -Last,S,`last_value`; `last`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -Max,S,`max`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS -Max,S,`max`,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS -Max,S,`max`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS -Max,S,`max`,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS -Max,S,`max`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS -Max,S,`max`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS -Min,S,`min`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS -Min,S,`min`,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS -Min,S,`min`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS -Min,S,`min`,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS -Min,S,`min`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS -Min,S,`min`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS -Percentile,S,`percentile`,None,aggregation,input,NA,S,S,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Percentile,S,`percentile`,None,aggregation,percentage,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA -Percentile,S,`percentile`,None,aggregation,frequency,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA -Percentile,S,`percentile`,None,aggregation,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA -Percentile,S,`percentile`,None,reduction,input,NA,S,S,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Percentile,S,`percentile`,None,reduction,percentage,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA -Percentile,S,`percentile`,None,reduction,frequency,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA -Percentile,S,`percentile`,None,reduction,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA -PivotFirst,S, ,None,aggregation,pivotColumn,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS -PivotFirst,S, ,None,aggregation,valueColumn,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS -PivotFirst,S, ,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,NS,NS -PivotFirst,S, ,None,reduction,pivotColumn,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS -PivotFirst,S, ,None,reduction,valueColumn,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS -PivotFirst,S, ,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,NS,NS -StddevPop,S,`stddev_pop`,None,reduction,input,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StddevPop,S,`stddev_pop`,None,reduction,result,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StddevPop,S,`stddev_pop`,None,aggregation,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StddevPop,S,`stddev_pop`,None,aggregation,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StddevPop,S,`stddev_pop`,None,window,input,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StddevPop,S,`stddev_pop`,None,window,result,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StddevSamp,S,`std`; `stddev_samp`; `stddev`,None,aggregation,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StddevSamp,S,`std`; `stddev_samp`; `stddev`,None,aggregation,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StddevSamp,S,`std`; `stddev_samp`; `stddev`,None,reduction,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StddevSamp,S,`std`; `stddev_samp`; `stddev`,None,reduction,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StddevSamp,S,`std`; `stddev_samp`; `stddev`,None,window,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StddevSamp,S,`std`; `stddev_samp`; `stddev`,None,window,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Sum,S,`sum`,None,aggregation,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Sum,S,`sum`,None,aggregation,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Sum,S,`sum`,None,reduction,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Sum,S,`sum`,None,reduction,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Sum,S,`sum`,None,window,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -Sum,S,`sum`,None,window,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA -VariancePop,S,`var_pop`,None,reduction,input,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -VariancePop,S,`var_pop`,None,reduction,result,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -VariancePop,S,`var_pop`,None,aggregation,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -VariancePop,S,`var_pop`,None,aggregation,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -VariancePop,S,`var_pop`,None,window,input,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -VariancePop,S,`var_pop`,None,window,result,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -VarianceSamp,S,`var_samp`; `variance`,None,reduction,input,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -VarianceSamp,S,`var_samp`; `variance`,None,reduction,result,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -VarianceSamp,S,`var_samp`; `variance`,None,aggregation,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -VarianceSamp,S,`var_samp`; `variance`,None,aggregation,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -VarianceSamp,S,`var_samp`; `variance`,None,window,input,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -VarianceSamp,S,`var_samp`; `variance`,None,window,result,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -NormalizeNaNAndZero,S, ,None,project,input,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -NormalizeNaNAndZero,S, ,None,project,result,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ScalarSubquery,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS -HiveGenericUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS -HiveGenericUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS -HiveSimpleUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS -HiveSimpleUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS +Expression,Supported,SQL Func,Notes,Context,Params,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH +Abs,S,`abs`,None,project,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Abs,S,`abs`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Abs,S,`abs`,None,AST,input,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA +Abs,S,`abs`,None,AST,result,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA +Acos,S,`acos`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Acos,S,`acos`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Acos,S,`acos`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Acos,S,`acos`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Acosh,S,`acosh`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Acosh,S,`acosh`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Acosh,S,`acosh`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Acosh,S,`acosh`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Add,S,`+`,None,project,lhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NS,NS +Add,S,`+`,None,project,rhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NS,NS +Add,S,`+`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NS,NS +Add,S,`+`,None,AST,lhs,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA,NS,NS +Add,S,`+`,None,AST,rhs,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA,NS,NS +Add,S,`+`,None,AST,result,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA,NS,NS +Alias,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +Alias,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +Alias,S, ,None,AST,input,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS +Alias,S, ,None,AST,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS +And,S,`and`,None,project,lhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +And,S,`and`,None,project,rhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +And,S,`and`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +And,S,`and`,None,AST,lhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +And,S,`and`,None,AST,rhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +And,S,`and`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ArrayContains,S,`array_contains`,None,project,array,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayContains,S,`array_contains`,None,project,key,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS +ArrayContains,S,`array_contains`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ArrayExcept,S,`array_except`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayExcept,S,`array_except`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayExcept,S,`array_except`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayExists,S,`exists`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayExists,S,`exists`,None,project,function,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ArrayExists,S,`exists`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ArrayFilter,S,`filter`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayFilter,S,`filter`,None,project,function,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ArrayFilter,S,`filter`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayIntersect,S,`array_intersect`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayIntersect,S,`array_intersect`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayIntersect,S,`array_intersect`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayMax,S,`array_max`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayMax,S,`array_max`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +ArrayMin,S,`array_min`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayMin,S,`array_min`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +ArrayRemove,S,`array_remove`,None,project,array,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,PS,NS,NS,NS,NS,NS +ArrayRemove,S,`array_remove`,None,project,element,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +ArrayRemove,S,`array_remove`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayRepeat,S,`array_repeat`,None,project,left,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +ArrayRepeat,S,`array_repeat`,None,project,right,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ArrayRepeat,S,`array_repeat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayTransform,S,`transform`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayTransform,S,`transform`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +ArrayTransform,S,`transform`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayUnion,S,`array_union`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayUnion,S,`array_union`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArrayUnion,S,`array_union`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArraysOverlap,S,`arrays_overlap`,This is not 100% compatible with the Spark version because the GPU implementation treats -0.0 and 0.0 as equal; but the CPU implementation currently does not (see SPARK-39845). Also; Apache Spark 3.1.3 fixed issue SPARK-36741 where NaNs in these set like operators were not treated as being equal. We have chosen to break with compatibility for the older versions of Spark in this instance and handle NaNs the same as 3.1.3+,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ArraysZip,S,`arrays_zip`,None,project,children,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ArraysZip,S,`arrays_zip`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +Ascii,NS,`ascii`,This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters after Spark 3.2.3; 3.3.1 and 3.4.0. Otherwise the results will not match the CPU.,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Ascii,NS,`ascii`,This is disabled by default because it only supports strings starting with ASCII or Latin-1 characters after Spark 3.2.3; 3.3.1 and 3.4.0. Otherwise the results will not match the CPU.,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Asin,S,`asin`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Asin,S,`asin`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Asin,S,`asin`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Asin,S,`asin`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Asinh,S,`asinh`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Asinh,S,`asinh`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Asinh,S,`asinh`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Asinh,S,`asinh`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +AtLeastNNonNulls,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +AtLeastNNonNulls,S, ,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Atan,S,`atan`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Atan,S,`atan`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Atan,S,`atan`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Atan,S,`atan`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Atanh,S,`atanh`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Atanh,S,`atanh`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Atanh,S,`atanh`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Atanh,S,`atanh`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +AttributeReference,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +AttributeReference,S, ,None,AST,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS +BRound,S,`bround`,None,project,value,NA,S,S,S,S,PS,PS,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +BRound,S,`bround`,None,project,scale,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BRound,S,`bround`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitLength,S,`bit_length`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA +BitLength,S,`bit_length`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseAnd,S,`&`,None,project,lhs,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseAnd,S,`&`,None,project,rhs,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseAnd,S,`&`,None,project,result,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseAnd,S,`&`,None,AST,lhs,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseAnd,S,`&`,None,AST,rhs,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseAnd,S,`&`,None,AST,result,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseNot,S,`~`,None,project,input,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseNot,S,`~`,None,project,result,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseNot,S,`~`,None,AST,input,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseNot,S,`~`,None,AST,result,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseOr,S,`\|`,None,project,lhs,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseOr,S,`\|`,None,project,rhs,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseOr,S,`\|`,None,project,result,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseOr,S,`\|`,None,AST,lhs,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseOr,S,`\|`,None,AST,rhs,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseOr,S,`\|`,None,AST,result,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseXor,S,`^`,None,project,lhs,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseXor,S,`^`,None,project,rhs,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseXor,S,`^`,None,project,result,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseXor,S,`^`,None,AST,lhs,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseXor,S,`^`,None,AST,rhs,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BitwiseXor,S,`^`,None,AST,result,NA,NS,NS,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +BoundReference,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +BoundReference,S, ,None,AST,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS +CaseWhen,S,`when`,None,project,predicate,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +CaseWhen,S,`when`,None,project,value,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +CaseWhen,S,`when`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +Cbrt,S,`cbrt`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cbrt,S,`cbrt`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cbrt,S,`cbrt`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cbrt,S,`cbrt`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Ceil,S,`ceil`; `ceiling`,None,project,input,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Ceil,S,`ceil`; `ceiling`,None,project,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +CheckOverflow,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +CheckOverflow,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Coalesce,S,`coalesce`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +Coalesce,S,`coalesce`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +Concat,S,`concat`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,PS,NA,NA,NA,NA,NA +Concat,S,`concat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,PS,NA,NA,NA,NA,NA +ConcatWs,S,`concat_ws`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,S,NA,NA,NA,NA,NA +ConcatWs,S,`concat_ws`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Contains,S, ,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Contains,S, ,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Contains,S, ,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Conv,NS,`conv`,This is disabled by default because GPU implementation is incomplete. We currently only support from/to_base values of 10 and 16. We fall back on CPU if the signed conversion is signalled via a negative to_base. GPU implementation does not check for an 64-bit signed/unsigned int overflow when performing the conversion to return `FFFFFFFFFFFFFFFF` or `18446744073709551615` or to throw an error in the ANSI mode. It is safe to enable if the overflow is not possible or detected externally. For instance decimal strings not longer than 18 characters / hexadecimal strings not longer than 15 characters disregarding the sign cannot cause an overflow. ,project,num,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Conv,NS,`conv`,This is disabled by default because GPU implementation is incomplete. We currently only support from/to_base values of 10 and 16. We fall back on CPU if the signed conversion is signalled via a negative to_base. GPU implementation does not check for an 64-bit signed/unsigned int overflow when performing the conversion to return `FFFFFFFFFFFFFFFF` or `18446744073709551615` or to throw an error in the ANSI mode. It is safe to enable if the overflow is not possible or detected externally. For instance decimal strings not longer than 18 characters / hexadecimal strings not longer than 15 characters disregarding the sign cannot cause an overflow. ,project,from_base,NA,PS,PS,PS,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Conv,NS,`conv`,This is disabled by default because GPU implementation is incomplete. We currently only support from/to_base values of 10 and 16. We fall back on CPU if the signed conversion is signalled via a negative to_base. GPU implementation does not check for an 64-bit signed/unsigned int overflow when performing the conversion to return `FFFFFFFFFFFFFFFF` or `18446744073709551615` or to throw an error in the ANSI mode. It is safe to enable if the overflow is not possible or detected externally. For instance decimal strings not longer than 18 characters / hexadecimal strings not longer than 15 characters disregarding the sign cannot cause an overflow. ,project,to_base,NA,PS,PS,PS,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Conv,NS,`conv`,This is disabled by default because GPU implementation is incomplete. We currently only support from/to_base values of 10 and 16. We fall back on CPU if the signed conversion is signalled via a negative to_base. GPU implementation does not check for an 64-bit signed/unsigned int overflow when performing the conversion to return `FFFFFFFFFFFFFFFF` or `18446744073709551615` or to throw an error in the ANSI mode. It is safe to enable if the overflow is not possible or detected externally. For instance decimal strings not longer than 18 characters / hexadecimal strings not longer than 15 characters disregarding the sign cannot cause an overflow. ,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cos,S,`cos`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cos,S,`cos`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cos,S,`cos`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cos,S,`cos`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cosh,S,`cosh`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cosh,S,`cosh`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cosh,S,`cosh`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cosh,S,`cosh`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cot,S,`cot`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cot,S,`cot`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cot,S,`cot`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Cot,S,`cot`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +CreateArray,S,`array`,None,project,arg,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,NS,PS,NS,NS,NS +CreateArray,S,`array`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +CreateMap,S,`map`,None,project,key,S,S,S,S,S,S,S,S,PS,S,S,S,NA,NA,PS,NA,PS,NA,NA,NA +CreateMap,S,`map`,None,project,value,S,S,S,S,S,S,S,S,PS,S,S,S,NA,NA,PS,PS,PS,NA,NA,NA +CreateNamedStruct,S,`named_struct`; `struct`,None,project,name,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +CreateNamedStruct,S,`named_struct`; `struct`,None,project,value,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +CreateNamedStruct,S,`named_struct`; `struct`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +CurrentRow$,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA +DateAdd,S,`date_add`,None,project,startDate,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateAdd,S,`date_add`,None,project,days,NA,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateAdd,S,`date_add`,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateAddInterval,S, ,None,project,start,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateAddInterval,S, ,None,project,interval,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA +DateAddInterval,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateDiff,S,`datediff`,None,project,lhs,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateDiff,S,`datediff`,None,project,rhs,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateDiff,S,`datediff`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateFormatClass,S,`date_format`,None,project,timestamp,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateFormatClass,S,`date_format`,None,project,strfmt,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateFormatClass,S,`date_format`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateSub,S,`date_sub`,None,project,startDate,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateSub,S,`date_sub`,None,project,days,NA,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateSub,S,`date_sub`,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DayOfMonth,S,`day`; `dayofmonth`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DayOfMonth,S,`day`; `dayofmonth`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DayOfWeek,S,`dayofweek`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DayOfWeek,S,`dayofweek`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DayOfYear,S,`dayofyear`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DayOfYear,S,`dayofyear`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DenseRank,S,`dense_rank`,None,window,ordering,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS,NS,NS +DenseRank,S,`dense_rank`,None,window,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Divide,S,`/`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Divide,S,`/`,None,project,rhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Divide,S,`/`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +DynamicPruningExpression,S, ,None,project,input,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DynamicPruningExpression,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +ElementAt,S,`element_at`,None,project,array/map,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,PS,NA,NA,NA,NA +ElementAt,S,`element_at`,None,project,index/key,PS,PS,PS,S,PS,PS,PS,PS,PS,PS,PS,NS,NS,NS,NS,NS,NS,NS,NS,NS +ElementAt,S,`element_at`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +EndsWith,S, ,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +EndsWith,S, ,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +EndsWith,S, ,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +EqualNullSafe,S,`<=>`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA +EqualNullSafe,S,`<=>`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA +EqualNullSafe,S,`<=>`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +EqualTo,S,`==`; `=`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA +EqualTo,S,`==`; `=`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA +EqualTo,S,`==`; `=`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +EqualTo,S,`==`; `=`,None,AST,lhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS,NA,NA +EqualTo,S,`==`; `=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS,NA,NA +EqualTo,S,`==`; `=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Exp,S,`exp`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Exp,S,`exp`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Exp,S,`exp`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Exp,S,`exp`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Explode,S,`explode_outer`; `explode`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,PS,NA,NA,NA,NA +Explode,S,`explode_outer`; `explode`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +Expm1,S,`expm1`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Expm1,S,`expm1`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Expm1,S,`expm1`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Expm1,S,`expm1`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Flatten,S,`flatten`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +Flatten,S,`flatten`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +Floor,S,`floor`,None,project,input,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Floor,S,`floor`,None,project,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +FormatNumber,S,`format_number`,None,project,x,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +FormatNumber,S,`format_number`,None,project,d,NA,NA,NA,PS,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +FormatNumber,S,`format_number`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +FromUTCTimestamp,S,`from_utc_timestamp`,None,project,timestamp,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +FromUTCTimestamp,S,`from_utc_timestamp`,None,project,timezone,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +FromUTCTimestamp,S,`from_utc_timestamp`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +FromUnixTime,S,`from_unixtime`,None,project,sec,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +FromUnixTime,S,`from_unixtime`,None,project,format,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +FromUnixTime,S,`from_unixtime`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +GetArrayItem,S, ,None,project,array,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +GetArrayItem,S, ,None,project,ordinal,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +GetArrayItem,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +GetArrayStructFields,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +GetArrayStructFields,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +GetJsonObject,NS,`get_json_object`,This is disabled by default because Experimental feature that could be unstable or have performance issues.,project,json,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +GetJsonObject,NS,`get_json_object`,This is disabled by default because Experimental feature that could be unstable or have performance issues.,project,path,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +GetJsonObject,NS,`get_json_object`,This is disabled by default because Experimental feature that could be unstable or have performance issues.,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +GetMapValue,S, ,None,project,map,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA +GetMapValue,S, ,None,project,key,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS +GetMapValue,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +GetStructField,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA +GetStructField,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +GetTimestamp,S, ,None,project,timeExp,NA,NA,NA,NA,NA,NA,NA,S,PS,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +GetTimestamp,S, ,None,project,format,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +GetTimestamp,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +GreaterThan,S,`>`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA +GreaterThan,S,`>`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA +GreaterThan,S,`>`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +GreaterThan,S,`>`,None,AST,lhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS,NA,NA +GreaterThan,S,`>`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS,NA,NA +GreaterThan,S,`>`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +GreaterThanOrEqual,S,`>=`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA +GreaterThanOrEqual,S,`>=`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA +GreaterThanOrEqual,S,`>=`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +GreaterThanOrEqual,S,`>=`,None,AST,lhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS,NA,NA +GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS,NA,NA +GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +HiveHash,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,NS,NS,NS,NS,NS,NS +HiveHash,S, ,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Hour,S,`hour`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Hypot,S,`hypot`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Hypot,S,`hypot`,None,project,rhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Hypot,S,`hypot`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +If,S,`if`,None,project,predicate,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +If,S,`if`,None,project,trueValue,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +If,S,`if`,None,project,falseValue,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +If,S,`if`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +In,S,`in`,None,project,value,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +In,S,`in`,None,project,list,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,NS,NS,NS,NS,NA,NS,NS,NA,NA +In,S,`in`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +InSet,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +InSet,S, ,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +InitCap,S,`initcap`,This is not 100% compatible with the Spark version because the Unicode version used by cuDF and the JVM may differ; resulting in some corner-case characters not changing case correctly.,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +InitCap,S,`initcap`,This is not 100% compatible with the Spark version because the Unicode version used by cuDF and the JVM may differ; resulting in some corner-case characters not changing case correctly.,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +InputFileBlockLength,S,`input_file_block_length`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +InputFileBlockStart,S,`input_file_block_start`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +InputFileName,S,`input_file_name`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +IntegralDivide,S,`div`,None,project,lhs,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +IntegralDivide,S,`div`,None,project,rhs,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +IntegralDivide,S,`div`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +IsNaN,S,`isnan`,None,project,input,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +IsNaN,S,`isnan`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +IsNotNull,S,`isnotnull`,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +IsNotNull,S,`isnotnull`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +IsNull,S,`isnull`,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +IsNull,S,`isnull`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +JsonToStructs,NS,`from_json`,This is disabled by default because it is currently in beta and undergoes continuous enhancements. Please consult the [compatibility documentation](../compatibility.md#json-supporting-types) to determine whether you can enable this configuration for your use case,project,jsonStr,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +JsonToStructs,NS,`from_json`,This is disabled by default because it is currently in beta and undergoes continuous enhancements. Please consult the [compatibility documentation](../compatibility.md#json-supporting-types) to determine whether you can enable this configuration for your use case,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NS,PS,PS,NA,NA,NA +JsonTuple,NS,`json_tuple`,This is disabled by default because Experimental feature that could be unstable or have performance issues.,project,json,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +JsonTuple,NS,`json_tuple`,This is disabled by default because Experimental feature that could be unstable or have performance issues.,project,field,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +JsonTuple,NS,`json_tuple`,This is disabled by default because Experimental feature that could be unstable or have performance issues.,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA +KnownFloatingPointNormalized,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +KnownFloatingPointNormalized,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +KnownNotNull,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,NS,S,S,PS,PS,PS,NS,NS,NS +KnownNotNull,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,NS,S,S,PS,PS,PS,NS,NS,NS +Lag,S,`lag`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS +Lag,S,`lag`,None,window,offset,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Lag,S,`lag`,None,window,default,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS +Lag,S,`lag`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS +LambdaFunction,S, ,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +LambdaFunction,S, ,None,project,arguments,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +LambdaFunction,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +LastDay,S,`last_day`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +LastDay,S,`last_day`,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Lead,S,`lead`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS +Lead,S,`lead`,None,window,offset,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Lead,S,`lead`,None,window,default,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS +Lead,S,`lead`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS +Least,S,`least`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +Least,S,`least`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +Length,S,`char_length`; `character_length`; `length`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA +Length,S,`char_length`; `character_length`; `length`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +LessThan,S,`<`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA +LessThan,S,`<`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA +LessThan,S,`<`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +LessThan,S,`<`,None,AST,lhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS,NA,NA +LessThan,S,`<`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS,NA,NA +LessThan,S,`<`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +LessThanOrEqual,S,`<=`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA +LessThanOrEqual,S,`<=`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA +LessThanOrEqual,S,`<=`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +LessThanOrEqual,S,`<=`,None,AST,lhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS,NA,NA +LessThanOrEqual,S,`<=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA,NS,NS,NA,NA +LessThanOrEqual,S,`<=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Like,S,`like`,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Like,S,`like`,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Like,S,`like`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Literal,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS,S,S +Literal,S, ,None,AST,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS +Log,S,`ln`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Log,S,`ln`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Log10,S,`log10`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Log10,S,`log10`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Log1p,S,`log1p`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Log1p,S,`log1p`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Log2,S,`log2`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Log2,S,`log2`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Logarithm,S,`log`,None,project,value,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Logarithm,S,`log`,None,project,base,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Logarithm,S,`log`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Lower,S,`lcase`; `lower`,This is not 100% compatible with the Spark version because the Unicode version used by cuDF and the JVM may differ; resulting in some corner-case characters not changing case correctly.,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Lower,S,`lcase`; `lower`,This is not 100% compatible with the Spark version because the Unicode version used by cuDF and the JVM may differ; resulting in some corner-case characters not changing case correctly.,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +MakeDecimal,S, ,None,project,input,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +MakeDecimal,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA +MapConcat,S,`map_concat`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA +MapConcat,S,`map_concat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA +MapEntries,S,`map_entries`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA +MapEntries,S,`map_entries`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +MapFilter,S,`map_filter`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA +MapFilter,S,`map_filter`,None,project,function,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +MapFilter,S,`map_filter`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA +MapKeys,S,`map_keys`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA +MapKeys,S,`map_keys`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +MapValues,S,`map_values`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA +MapValues,S,`map_values`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +Md5,S,`md5`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA +Md5,S,`md5`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +MicrosToTimestamp,S,`timestamp_micros`,None,project,input,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +MicrosToTimestamp,S,`timestamp_micros`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +MillisToTimestamp,S,`timestamp_millis`,None,project,input,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +MillisToTimestamp,S,`timestamp_millis`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Minute,S,`minute`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Minute,S,`minute`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +MonotonicallyIncreasingID,S,`monotonically_increasing_id`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Month,S,`month`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Month,S,`month`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Multiply,S,`*`,None,project,lhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Multiply,S,`*`,None,project,rhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Multiply,S,`*`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Multiply,S,`*`,None,AST,lhs,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA +Multiply,S,`*`,None,AST,rhs,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA +Multiply,S,`*`,None,AST,result,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA +Murmur3Hash,S,`hash`,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS +Murmur3Hash,S,`hash`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +NaNvl,S,`nanvl`,None,project,lhs,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +NaNvl,S,`nanvl`,None,project,rhs,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +NaNvl,S,`nanvl`,None,project,result,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +NamedLambdaVariable,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +Not,S,`!`; `not`,None,project,input,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Not,S,`!`; `not`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Not,S,`!`; `not`,None,AST,input,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Not,S,`!`; `not`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +NthValue,S,`nth_value`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +NthValue,S,`nth_value`,None,window,offset,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +NthValue,S,`nth_value`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +OctetLength,S,`octet_length`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA +OctetLength,S,`octet_length`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Or,S,`or`,None,project,lhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Or,S,`or`,None,project,rhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Or,S,`or`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Or,S,`or`,None,AST,lhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Or,S,`or`,None,AST,rhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Or,S,`or`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ParseUrl,S,`parse_url`,None,project,url,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ParseUrl,S,`parse_url`,None,project,partToExtract,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ParseUrl,S,`parse_url`,None,project,key,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ParseUrl,S,`parse_url`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +PercentRank,S,`percent_rank`,None,window,ordering,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS,NS,NS +PercentRank,S,`percent_rank`,None,window,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Pmod,S,`pmod`,None,project,lhs,NA,S,S,S,S,S,S,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA +Pmod,S,`pmod`,None,project,rhs,NA,S,S,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA +Pmod,S,`pmod`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA +PosExplode,S,`posexplode_outer`; `posexplode`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,PS,NA,NA,NA,NA +PosExplode,S,`posexplode_outer`; `posexplode`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +Pow,S,`pow`; `power`,None,project,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Pow,S,`pow`; `power`,None,project,rhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Pow,S,`pow`; `power`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Pow,S,`pow`; `power`,None,AST,lhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Pow,S,`pow`; `power`,None,AST,rhs,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Pow,S,`pow`; `power`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +PreciseTimestampConversion,S, ,None,project,input,NA,NA,NA,NA,S,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +PreciseTimestampConversion,S, ,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +PromotePrecision,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +PromotePrecision,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +PythonUDF,S, ,None,aggregation,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS +PythonUDF,S, ,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA,NA,NA +PythonUDF,S, ,None,reduction,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS +PythonUDF,S, ,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA,NA,NA +PythonUDF,S, ,None,window,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS +PythonUDF,S, ,None,window,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA,NA,NA +PythonUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,PS,NS,PS,NS,NS,NS +PythonUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NA,PS,NS,PS,NA,NA,NA +Quarter,S,`quarter`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Quarter,S,`quarter`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RLike,S,`regexp_like`; `regexp`; `rlike`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RLike,S,`regexp_like`; `regexp`; `rlike`,None,project,regexp,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RLike,S,`regexp_like`; `regexp`; `rlike`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RaiseError,S,`raise_error`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RaiseError,S,`raise_error`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA +Rand,S,`rand`; `random`,None,project,seed,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Rand,S,`rand`; `random`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Rank,S,`rank`,None,window,ordering,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS,NS,NS +Rank,S,`rank`,None,window,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RegExpExtract,S,`regexp_extract`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RegExpExtract,S,`regexp_extract`,None,project,regexp,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RegExpExtract,S,`regexp_extract`,None,project,idx,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RegExpExtract,S,`regexp_extract`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RegExpExtractAll,S,`regexp_extract_all`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RegExpExtractAll,S,`regexp_extract_all`,None,project,regexp,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RegExpExtractAll,S,`regexp_extract_all`,None,project,idx,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RegExpExtractAll,S,`regexp_extract_all`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA +RegExpReplace,S,`regexp_replace`,None,project,regex,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RegExpReplace,S,`regexp_replace`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RegExpReplace,S,`regexp_replace`,None,project,pos,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RegExpReplace,S,`regexp_replace`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +RegExpReplace,S,`regexp_replace`,None,project,rep,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Remainder,S,`%`; `mod`,None,project,lhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Remainder,S,`%`; `mod`,None,project,rhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Remainder,S,`%`; `mod`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +ReplicateRows,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS +ReplicateRows,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +Reverse,S,`reverse`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +Reverse,S,`reverse`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +Rint,S,`rint`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Rint,S,`rint`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Rint,S,`rint`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Rint,S,`rint`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Round,S,`round`,None,project,value,NA,S,S,S,S,PS,PS,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Round,S,`round`,None,project,scale,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Round,S,`round`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +RowNumber,S,`row_number`,None,window,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ScalaUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS,NS,NS +ScalaUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS,NS,NS +Second,S,`second`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Second,S,`second`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +SecondsToTimestamp,S,`timestamp_seconds`,None,project,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +SecondsToTimestamp,S,`timestamp_seconds`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sequence,S,`sequence`,None,project,start,NA,S,S,S,S,NA,NA,NS,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sequence,S,`sequence`,None,project,stop,NA,S,S,S,S,NA,NA,NS,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sequence,S,`sequence`,None,project,step,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA +Sequence,S,`sequence`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +ShiftLeft,S,`shiftleft`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftLeft,S,`shiftleft`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftLeft,S,`shiftleft`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRight,S,`shiftright`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRight,S,`shiftright`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRight,S,`shiftright`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRightUnsigned,S,`shiftrightunsigned`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRightUnsigned,S,`shiftrightunsigned`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRightUnsigned,S,`shiftrightunsigned`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Signum,S,`sign`; `signum`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Signum,S,`sign`; `signum`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sin,S,`sin`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sin,S,`sin`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sin,S,`sin`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sin,S,`sin`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sinh,S,`sinh`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sinh,S,`sinh`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sinh,S,`sinh`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sinh,S,`sinh`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Size,S,`cardinality`; `size`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,PS,NA,NA,NA,NA +Size,S,`cardinality`; `size`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +SortArray,S,`sort_array`,None,project,array,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +SortArray,S,`sort_array`,None,project,ascendingOrder,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +SortArray,S,`sort_array`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +SortOrder,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS,NA,NA +SortOrder,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS,NA,NA +SparkPartitionID,S,`spark_partition_id`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +SpecifiedWindowFrame,S, ,None,project,lower,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,S,NA,NA,NA,NA,S,NS +SpecifiedWindowFrame,S, ,None,project,upper,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,S,NA,NA,NA,NA,S,NS +SpecifiedWindowFrame,S, ,None,project,result,NA,S,S,S,S,NS,NS,NA,NA,NA,NS,NA,NA,S,NA,NA,NA,NA,S,NS +Sqrt,S,`sqrt`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sqrt,S,`sqrt`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sqrt,S,`sqrt`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sqrt,S,`sqrt`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Stack,S,`stack`,None,project,n,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Stack,S,`stack`,None,project,expr,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +Stack,S,`stack`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +StartsWith,S, ,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StartsWith,S, ,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StartsWith,S, ,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringInstr,S,`instr`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringInstr,S,`instr`,None,project,substr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringInstr,S,`instr`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLPad,S,`lpad`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLPad,S,`lpad`,None,project,len,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLPad,S,`lpad`,None,project,pad,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLPad,S,`lpad`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLocate,S,`locate`; `position`,None,project,substr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLocate,S,`locate`; `position`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLocate,S,`locate`; `position`,None,project,start,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLocate,S,`locate`; `position`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRPad,S,`rpad`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRPad,S,`rpad`,None,project,len,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRPad,S,`rpad`,None,project,pad,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRPad,S,`rpad`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRepeat,S,`repeat`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRepeat,S,`repeat`,None,project,repeatTimes,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRepeat,S,`repeat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringReplace,S,`replace`,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringReplace,S,`replace`,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringReplace,S,`replace`,None,project,replace,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringReplace,S,`replace`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringSplit,S,`split`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringSplit,S,`split`,None,project,regexp,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringSplit,S,`split`,None,project,limit,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringSplit,S,`split`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA +StringToMap,S,`str_to_map`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringToMap,S,`str_to_map`,None,project,pairDelim,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringToMap,S,`str_to_map`,None,project,keyValueDelim,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringToMap,S,`str_to_map`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA +StringTranslate,S,`translate`,This is not 100% compatible with the Spark version because the GPU implementation supports all unicode code points. In Spark versions < 3.2.0; translate() does not support unicode characters with code point >= U+10000 (See SPARK-34094),project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringTranslate,S,`translate`,This is not 100% compatible with the Spark version because the GPU implementation supports all unicode code points. In Spark versions < 3.2.0; translate() does not support unicode characters with code point >= U+10000 (See SPARK-34094),project,from,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringTranslate,S,`translate`,This is not 100% compatible with the Spark version because the GPU implementation supports all unicode code points. In Spark versions < 3.2.0; translate() does not support unicode characters with code point >= U+10000 (See SPARK-34094),project,to,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringTranslate,S,`translate`,This is not 100% compatible with the Spark version because the GPU implementation supports all unicode code points. In Spark versions < 3.2.0; translate() does not support unicode characters with code point >= U+10000 (See SPARK-34094),project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringTrim,S,`trim`,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringTrim,S,`trim`,None,project,trimStr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringTrim,S,`trim`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringTrimLeft,S,`ltrim`,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringTrimLeft,S,`ltrim`,None,project,trimStr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringTrimLeft,S,`ltrim`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringTrimRight,S,`rtrim`,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringTrimRight,S,`rtrim`,None,project,trimStr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringTrimRight,S,`rtrim`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StructsToJson,NS,`to_json`,This is disabled by default because it is currently in beta and undergoes continuous enhancements. Please consult the [compatibility documentation](../compatibility.md#json-supporting-types) to determine whether you can enable this configuration for your use case,project,struct,S,S,S,S,S,S,S,S,PS,S,S,NA,NA,NA,PS,PS,PS,NA,NA,NA +StructsToJson,NS,`to_json`,This is disabled by default because it is currently in beta and undergoes continuous enhancements. Please consult the [compatibility documentation](../compatibility.md#json-supporting-types) to determine whether you can enable this configuration for your use case,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Substring,S,`substr`; `substring`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA +Substring,S,`substr`; `substring`,None,project,pos,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Substring,S,`substr`; `substring`,None,project,len,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Substring,S,`substr`; `substring`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA +SubstringIndex,S,`substring_index`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +SubstringIndex,S,`substring_index`,None,project,delim,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +SubstringIndex,S,`substring_index`,None,project,count,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +SubstringIndex,S,`substring_index`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Subtract,S,`-`,None,project,lhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NS,NS +Subtract,S,`-`,None,project,rhs,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NS,NS +Subtract,S,`-`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NS,NS +Subtract,S,`-`,None,AST,lhs,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA,NS,NS +Subtract,S,`-`,None,AST,rhs,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA,NS,NS +Subtract,S,`-`,None,AST,result,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA,NS,NS +Tan,S,`tan`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Tan,S,`tan`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Tan,S,`tan`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Tan,S,`tan`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Tanh,S,`tanh`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Tanh,S,`tanh`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Tanh,S,`tanh`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Tanh,S,`tanh`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +TimeAdd,S, ,None,project,start,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +TimeAdd,S, ,None,project,interval,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,PS,NA +TimeAdd,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ToDegrees,S,`degrees`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ToDegrees,S,`degrees`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ToRadians,S,`radians`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ToRadians,S,`radians`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ToUTCTimestamp,S,`to_utc_timestamp`,None,project,timestamp,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ToUTCTimestamp,S,`to_utc_timestamp`,None,project,timezone,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ToUTCTimestamp,S,`to_utc_timestamp`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ToUnixTimestamp,S,`to_unix_timestamp`,None,project,timeExp,NA,NA,NA,NA,NA,NA,NA,S,PS,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ToUnixTimestamp,S,`to_unix_timestamp`,None,project,format,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ToUnixTimestamp,S,`to_unix_timestamp`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +TransformKeys,S,`transform_keys`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA +TransformKeys,S,`transform_keys`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NS,NS +TransformKeys,S,`transform_keys`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA +TransformValues,S,`transform_values`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA +TransformValues,S,`transform_values`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +TransformValues,S,`transform_values`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA +UnaryMinus,S,`negative`,None,project,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NS,NS +UnaryMinus,S,`negative`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NS,NS +UnaryMinus,S,`negative`,None,AST,input,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA,NS,NS +UnaryMinus,S,`negative`,None,AST,result,NA,NS,NS,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA,NS,NS +UnaryPositive,S,`positive`,None,project,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NS,NS +UnaryPositive,S,`positive`,None,project,result,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NS,NS +UnaryPositive,S,`positive`,None,AST,input,NA,S,S,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA,NS,NS +UnaryPositive,S,`positive`,None,AST,result,NA,S,S,S,S,S,S,NA,NA,NA,NS,NA,NA,NS,NA,NA,NA,NA,NS,NS +UnboundedFollowing$,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA +UnboundedPreceding$,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA +UnixTimestamp,S,`unix_timestamp`,None,project,timeExp,NA,NA,NA,NA,NA,NA,NA,S,PS,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +UnixTimestamp,S,`unix_timestamp`,None,project,format,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +UnixTimestamp,S,`unix_timestamp`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +UnscaledValue,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA +UnscaledValue,S, ,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Upper,S,`ucase`; `upper`,This is not 100% compatible with the Spark version because the Unicode version used by cuDF and the JVM may differ; resulting in some corner-case characters not changing case correctly.,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Upper,S,`ucase`; `upper`,This is not 100% compatible with the Spark version because the Unicode version used by cuDF and the JVM may differ; resulting in some corner-case characters not changing case correctly.,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +WeekDay,S,`weekday`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +WeekDay,S,`weekday`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +WindowExpression,S, ,None,window,windowFunction,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +WindowExpression,S, ,None,window,windowSpec,NA,S,S,S,S,NS,NS,NA,NA,NA,PS,NA,NA,S,NA,NA,NA,NA,S,NS +WindowExpression,S, ,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +WindowSpecDefinition,S, ,None,project,partition,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,PS,NS,NS,NS +WindowSpecDefinition,S, ,None,project,value,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,PS,NS,NS,NS +WindowSpecDefinition,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,PS,NS,NS,NS +XxHash64,S,`xxhash64`,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS,NS,NS +XxHash64,S,`xxhash64`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Year,S,`year`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Year,S,`year`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +AggregateExpression,S, ,None,aggregation,aggFunc,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +AggregateExpression,S, ,None,aggregation,filter,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +AggregateExpression,S, ,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +AggregateExpression,S, ,None,reduction,aggFunc,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +AggregateExpression,S, ,None,reduction,filter,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +AggregateExpression,S, ,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +AggregateExpression,S, ,None,window,aggFunc,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +AggregateExpression,S, ,None,window,filter,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +AggregateExpression,S, ,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,aggregation,input,NA,S,S,S,S,S,S,NS,NS,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,aggregation,percentage,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA +ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,aggregation,accuracy,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,aggregation,result,NA,S,S,S,S,S,S,NS,NS,NA,S,NA,NA,NA,PS,NA,NA,NA,NA,NA +ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,reduction,input,NA,S,S,S,S,S,S,NS,NS,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,reduction,percentage,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA +ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,reduction,accuracy,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ApproximatePercentile,S,`approx_percentile`; `percentile_approx`,This is not 100% compatible with the Spark version because the GPU implementation of approx_percentile is not bit-for-bit compatible with Apache Spark,reduction,result,NA,S,S,S,S,S,S,NS,NS,NA,S,NA,NA,NA,PS,NA,NA,NA,NA,NA +Average,S,`avg`; `mean`,None,aggregation,input,NA,S,S,S,S,S,S,NA,NA,NA,S,S,NA,NS,NA,NA,NA,NA,NS,NS +Average,S,`avg`; `mean`,None,aggregation,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Average,S,`avg`; `mean`,None,reduction,input,NA,S,S,S,S,S,S,NA,NA,NA,S,S,NA,NS,NA,NA,NA,NA,NS,NS +Average,S,`avg`; `mean`,None,reduction,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Average,S,`avg`; `mean`,None,window,input,NA,S,S,S,S,S,S,NA,NA,NA,S,S,NA,NS,NA,NA,NA,NA,NS,NS +Average,S,`avg`; `mean`,None,window,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +CollectList,S,`collect_list`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +CollectList,S,`collect_list`,None,aggregation,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +CollectList,S,`collect_list`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +CollectList,S,`collect_list`,None,reduction,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +CollectList,S,`collect_list`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +CollectList,S,`collect_list`,None,window,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +CollectSet,S,`collect_set`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS +CollectSet,S,`collect_set`,None,aggregation,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +CollectSet,S,`collect_set`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS +CollectSet,S,`collect_set`,None,reduction,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +CollectSet,S,`collect_set`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS +CollectSet,S,`collect_set`,None,window,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA +Count,S,`count`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +Count,S,`count`,None,aggregation,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Count,S,`count`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +Count,S,`count`,None,reduction,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Count,S,`count`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +Count,S,`count`,None,window,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +First,S,`first_value`; `first`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +First,S,`first_value`; `first`,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +First,S,`first_value`; `first`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +First,S,`first_value`; `first`,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +First,S,`first_value`; `first`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +First,S,`first_value`; `first`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +Last,S,`last_value`; `last`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +Last,S,`last_value`; `last`,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +Last,S,`last_value`; `last`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +Last,S,`last_value`; `last`,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +Last,S,`last_value`; `last`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +Last,S,`last_value`; `last`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +Max,S,`max`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS,NA,NA +Max,S,`max`,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS,NA,NA +Max,S,`max`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS,NA,NA +Max,S,`max`,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS,NA,NA +Max,S,`max`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +Max,S,`max`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +Min,S,`min`,None,aggregation,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS,NA,NA +Min,S,`min`,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS,NA,NA +Min,S,`min`,None,reduction,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS,NA,NA +Min,S,`min`,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NA,PS,NS,NA,NA +Min,S,`min`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +Min,S,`min`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +Percentile,S,`percentile`,None,aggregation,input,NA,S,S,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Percentile,S,`percentile`,None,aggregation,percentage,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA +Percentile,S,`percentile`,None,aggregation,frequency,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA +Percentile,S,`percentile`,None,aggregation,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA +Percentile,S,`percentile`,None,reduction,input,NA,S,S,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Percentile,S,`percentile`,None,reduction,percentage,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA +Percentile,S,`percentile`,None,reduction,frequency,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA +Percentile,S,`percentile`,None,reduction,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA +PivotFirst,S, ,None,aggregation,pivotColumn,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS,NS,NS +PivotFirst,S, ,None,aggregation,valueColumn,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS,NS,NS +PivotFirst,S, ,None,aggregation,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,NS,NS,NS,NS +PivotFirst,S, ,None,reduction,pivotColumn,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS,NS,NS +PivotFirst,S, ,None,reduction,valueColumn,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,NS,NS,NS,NS +PivotFirst,S, ,None,reduction,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,NS,NS,NS,NS +StddevPop,S,`stddev_pop`,None,reduction,input,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StddevPop,S,`stddev_pop`,None,reduction,result,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StddevPop,S,`stddev_pop`,None,aggregation,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StddevPop,S,`stddev_pop`,None,aggregation,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StddevPop,S,`stddev_pop`,None,window,input,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StddevPop,S,`stddev_pop`,None,window,result,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StddevSamp,S,`std`; `stddev_samp`; `stddev`,None,aggregation,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StddevSamp,S,`std`; `stddev_samp`; `stddev`,None,aggregation,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StddevSamp,S,`std`; `stddev_samp`; `stddev`,None,reduction,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StddevSamp,S,`std`; `stddev_samp`; `stddev`,None,reduction,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StddevSamp,S,`std`; `stddev_samp`; `stddev`,None,window,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StddevSamp,S,`std`; `stddev_samp`; `stddev`,None,window,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sum,S,`sum`,None,aggregation,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sum,S,`sum`,None,aggregation,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sum,S,`sum`,None,reduction,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sum,S,`sum`,None,reduction,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sum,S,`sum`,None,window,input,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sum,S,`sum`,None,window,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +VariancePop,S,`var_pop`,None,reduction,input,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +VariancePop,S,`var_pop`,None,reduction,result,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +VariancePop,S,`var_pop`,None,aggregation,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +VariancePop,S,`var_pop`,None,aggregation,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +VariancePop,S,`var_pop`,None,window,input,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +VariancePop,S,`var_pop`,None,window,result,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +VarianceSamp,S,`var_samp`; `variance`,None,reduction,input,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +VarianceSamp,S,`var_samp`; `variance`,None,reduction,result,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +VarianceSamp,S,`var_samp`; `variance`,None,aggregation,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +VarianceSamp,S,`var_samp`; `variance`,None,aggregation,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +VarianceSamp,S,`var_samp`; `variance`,None,window,input,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +VarianceSamp,S,`var_samp`; `variance`,None,window,result,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +NormalizeNaNAndZero,S, ,None,project,input,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +NormalizeNaNAndZero,S, ,None,project,result,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ScalarSubquery,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +HiveGenericUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS,NS,NS +HiveGenericUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS,NS,NS +HiveSimpleUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS,NS,NS +HiveSimpleUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS,NS,NS From 3c89a31301b87f5e192dce3be43230c87758e526 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Fri, 12 Jul 2024 16:11:10 -0500 Subject: [PATCH 78/79] Avoid listFiles or inputFiles on relations with static partitioning (#11170) Signed-off-by: Jason Lowe --- .../src/main/python/delta_lake_write_test.py | 24 +++++++++ .../nvidia/spark/rapids/GpuOverrides.scala | 10 +++- .../sql/rapids/GpuFileSourceScanExec.scala | 22 ++++----- .../rapids/shims/StaticPartitionShims.scala | 49 +++++++++++++++++++ .../rapids/shims/StaticPartitionShims.scala | 35 +++++++++++++ 5 files changed, 126 insertions(+), 14 deletions(-) create mode 100644 sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/StaticPartitionShims.scala create mode 100644 sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/StaticPartitionShims.scala diff --git a/integration_tests/src/main/python/delta_lake_write_test.py b/integration_tests/src/main/python/delta_lake_write_test.py index d94e665aec0..c2292fe7c15 100644 --- a/integration_tests/src/main/python/delta_lake_write_test.py +++ b/integration_tests/src/main/python/delta_lake_write_test.py @@ -1049,3 +1049,27 @@ def test_delta_write_column_name_mapping(spark_tmp_path, mapping): lambda spark, path: spark.read.format("delta").load(path), data_path, conf=confs) + +# Hash aggregate can be used in a metadata query for compaction which completely falls back +compaction_allow = "HashAggregateExec" +if is_databricks_runtime(): + # compaction can fallback due to unsupported WriteIntoDeltaCommand + # tracked by https://github.com/NVIDIA/spark-rapids/issues/11169 + compaction_allow += "," + delta_write_fallback_allow +@allow_non_gpu(compaction_allow, *delta_meta_allow) +@delta_lake +@ignore_order +def test_delta_compaction(spark_tmp_path): + from delta.tables import DeltaTable + def do_write(spark, path): + spark.range(1000).write.mode("append").format("delta").save(path) + DeltaTable.forPath(spark, path).optimize().executeCompaction() + data_path = spark_tmp_path + "/DELTA_DATA" + confs = _delta_confs + with_cpu_session( + lambda spark: _create_cpu_gpu_tables(spark, data_path, "id bigint"), conf=confs) + assert_gpu_and_cpu_writes_are_equal_collect( + do_write, + lambda spark, path: spark.read.format("delta").load(path), + data_path, + conf=confs) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index f7b71d96ce6..cf77e25c04a 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -4646,8 +4646,14 @@ case class GpuOverrides() extends Rule[SparkPlan] with Logging { } // example filename: "file:/tmp/delta-table/_delta_log/00000000000000000000.json" - val found = f.relation.inputFiles.exists { name => - checkDeltaFunc(name) + val found = StaticPartitionShims.getStaticPartitions(f.relation).map { parts => + parts.exists { part => + part.files.exists(partFile => checkDeltaFunc(partFile.filePath.toString)) + } + }.getOrElse { + f.relation.inputFiles.exists { name => + checkDeltaFunc(name) + } } if (found) { logDebug(s"Fallback for FileSourceScanExec delta log: $f") diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileSourceScanExec.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileSourceScanExec.scala index 7195580ba69..d3f6e6dc1b3 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileSourceScanExec.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileSourceScanExec.scala @@ -22,7 +22,7 @@ import scala.collection.mutable.HashMap import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.filecache.FileCacheLocalityManager -import com.nvidia.spark.rapids.shims.{GpuDataSourceRDD, PartitionedFileUtilsShim, SparkShimImpl} +import com.nvidia.spark.rapids.shims.{GpuDataSourceRDD, PartitionedFileUtilsShim, SparkShimImpl, StaticPartitionShims} import org.apache.hadoop.fs.Path import org.apache.spark.rdd.RDD @@ -380,7 +380,7 @@ case class GpuFileSourceScanExec( createBucketedReadRDD(relation.bucketSpec.get, readFile, dynamicallySelectedPartitions, relation) } else { - createNonBucketedReadRDD(readFile, dynamicallySelectedPartitions, relation) + createNonBucketedReadRDD(readFile, relation) } sendDriverMetrics() readRDD @@ -550,24 +550,22 @@ case class GpuFileSourceScanExec( * * @param readFile an optional function to read each (part of a) file. Used when * not using the small file optimization. - * @param selectedPartitions Hive-style partition that are part of the read. * @param fsRelation [[HadoopFsRelation]] associated with the read. */ private def createNonBucketedReadRDD( readFile: Option[(PartitionedFile) => Iterator[InternalRow]], - selectedPartitions: Array[PartitionDirectory], fsRelation: HadoopFsRelation): RDD[InternalRow] = { - val openCostInBytes = fsRelation.sparkSession.sessionState.conf.filesOpenCostInBytes - val maxSplitBytes = - FilePartition.maxSplitBytes(fsRelation.sparkSession, selectedPartitions) - logInfo(s"Planning scan with bin packing, max size: $maxSplitBytes bytes, " + - s"open cost is considered as scanning $openCostInBytes bytes.") + val partitions = StaticPartitionShims.getStaticPartitions(fsRelation).getOrElse { + val openCostInBytes = fsRelation.sparkSession.sessionState.conf.filesOpenCostInBytes + val maxSplitBytes = + FilePartition.maxSplitBytes(fsRelation.sparkSession, selectedPartitions) + logInfo(s"Planning scan with bin packing, max size: $maxSplitBytes bytes, " + + s"open cost is considered as scanning $openCostInBytes bytes.") - val splitFiles = FilePartitionShims.splitFiles(selectedPartitions, relation, maxSplitBytes) + val splitFiles = FilePartitionShims.splitFiles(selectedPartitions, relation, maxSplitBytes) - val partitions = FilePartition.getFilePartitions(relation.sparkSession, splitFiles, maxSplitBytes) - + } getFinalRDD(readFile, partitions) } diff --git a/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/StaticPartitionShims.scala b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/StaticPartitionShims.scala new file mode 100644 index 00000000000..866d7ff440f --- /dev/null +++ b/sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/StaticPartitionShims.scala @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import org.apache.spark.sql.execution.datasources.FilePartition +import org.apache.spark.sql.execution.datasources.HadoopFsRelation + +object StaticPartitionShims { + /** Get the static partitions associated with a relation, if any. */ + def getStaticPartitions(relation: HadoopFsRelation): Option[Seq[FilePartition]] = None +} diff --git a/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/StaticPartitionShims.scala b/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/StaticPartitionShims.scala new file mode 100644 index 00000000000..b2ebcd50617 --- /dev/null +++ b/sql-plugin/src/main/spark341db/scala/com/nvidia/spark/rapids/shims/StaticPartitionShims.scala @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "341db"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import com.databricks.sql.transaction.tahoe.files.TahoeFileIndexWithStaticPartitions + +import org.apache.spark.sql.execution.datasources.FilePartition +import org.apache.spark.sql.execution.datasources.HadoopFsRelation + +object StaticPartitionShims { + /** Get the static partitions associated with a relation, if any. */ + def getStaticPartitions(relation: HadoopFsRelation): Option[Seq[FilePartition]] = { + relation.location match { + case t: TahoeFileIndexWithStaticPartitions => Some(t.getStaticPartitions) + case _ => None + } + } +} From 98917ebddd6012801fbb4afa5134fe0278761022 Mon Sep 17 00:00:00 2001 From: nvauto <70000568+nvauto@users.noreply.github.com> Date: Sun, 14 Jul 2024 03:48:25 +0000 Subject: [PATCH 79/79] Change version to 24.08.0 Signed-off-by: nvauto <70000568+nvauto@users.noreply.github.com> --- CONTRIBUTING.md | 8 ++++---- README.md | 2 +- aggregator/pom.xml | 4 ++-- api_validation/pom.xml | 4 ++-- datagen/ScaleTest.md | 2 +- datagen/pom.xml | 4 ++-- delta-lake/delta-20x/pom.xml | 4 ++-- delta-lake/delta-21x/pom.xml | 4 ++-- delta-lake/delta-22x/pom.xml | 4 ++-- delta-lake/delta-23x/pom.xml | 4 ++-- delta-lake/delta-24x/pom.xml | 4 ++-- delta-lake/delta-spark330db/pom.xml | 4 ++-- delta-lake/delta-spark332db/pom.xml | 4 ++-- delta-lake/delta-spark341db/pom.xml | 4 ++-- delta-lake/delta-stub/pom.xml | 4 ++-- dist/pom.xml | 4 ++-- docs/configs.md | 2 +- integration_tests/README.md | 6 +++--- integration_tests/ScaleTest.md | 2 +- integration_tests/pom.xml | 4 ++-- jdk-profiles/pom.xml | 4 ++-- jenkins/databricks/create.py | 2 +- jenkins/version-def.sh | 6 +++--- pom.xml | 2 +- scala2.13/aggregator/pom.xml | 4 ++-- scala2.13/api_validation/pom.xml | 4 ++-- scala2.13/datagen/pom.xml | 4 ++-- scala2.13/delta-lake/delta-20x/pom.xml | 4 ++-- scala2.13/delta-lake/delta-21x/pom.xml | 4 ++-- scala2.13/delta-lake/delta-22x/pom.xml | 4 ++-- scala2.13/delta-lake/delta-23x/pom.xml | 4 ++-- scala2.13/delta-lake/delta-24x/pom.xml | 4 ++-- scala2.13/delta-lake/delta-spark330db/pom.xml | 4 ++-- scala2.13/delta-lake/delta-spark332db/pom.xml | 4 ++-- scala2.13/delta-lake/delta-spark341db/pom.xml | 4 ++-- scala2.13/delta-lake/delta-stub/pom.xml | 4 ++-- scala2.13/dist/pom.xml | 4 ++-- scala2.13/integration_tests/pom.xml | 4 ++-- scala2.13/jdk-profiles/pom.xml | 4 ++-- scala2.13/pom.xml | 2 +- scala2.13/shim-deps/cloudera/pom.xml | 4 ++-- scala2.13/shim-deps/databricks/pom.xml | 4 ++-- scala2.13/shim-deps/pom.xml | 4 ++-- scala2.13/shuffle-plugin/pom.xml | 4 ++-- scala2.13/sql-plugin-api/pom.xml | 4 ++-- scala2.13/sql-plugin/pom.xml | 4 ++-- scala2.13/tests/pom.xml | 4 ++-- scala2.13/tools/pom.xml | 4 ++-- scala2.13/udf-compiler/pom.xml | 4 ++-- shim-deps/cloudera/pom.xml | 4 ++-- shim-deps/databricks/pom.xml | 4 ++-- shim-deps/pom.xml | 4 ++-- shuffle-plugin/pom.xml | 4 ++-- sql-plugin-api/pom.xml | 4 ++-- sql-plugin/pom.xml | 4 ++-- .../main/scala/com/nvidia/spark/rapids/RapidsConf.scala | 2 +- tests/pom.xml | 4 ++-- tools/pom.xml | 4 ++-- udf-compiler/pom.xml | 4 ++-- 59 files changed, 114 insertions(+), 114 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c52516023f1..1636211f784 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -127,15 +127,15 @@ mvn -pl dist -PnoSnapshots package -DskipTests Verify that shim-specific classes are hidden from a conventional classloader. ```bash -$ javap -cp dist/target/rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar com.nvidia.spark.rapids.shims.SparkShimImpl +$ javap -cp dist/target/rapids-4-spark_2.12-24.08.0-cuda11.jar com.nvidia.spark.rapids.shims.SparkShimImpl Error: class not found: com.nvidia.spark.rapids.shims.SparkShimImpl ``` However, its bytecode can be loaded if prefixed with `spark3XY` not contained in the package name ```bash -$ javap -cp dist/target/rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar spark320.com.nvidia.spark.rapids.shims.SparkShimImpl | head -2 -Warning: File dist/target/rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar(/spark320/com/nvidia/spark/rapids/shims/SparkShimImpl.class) does not contain class spark320.com.nvidia.spark.rapids.shims.SparkShimImpl +$ javap -cp dist/target/rapids-4-spark_2.12-24.08.0-cuda11.jar spark320.com.nvidia.spark.rapids.shims.SparkShimImpl | head -2 +Warning: File dist/target/rapids-4-spark_2.12-24.08.0-cuda11.jar(/spark320/com/nvidia/spark/rapids/shims/SparkShimImpl.class) does not contain class spark320.com.nvidia.spark.rapids.shims.SparkShimImpl Compiled from "SparkShims.scala" public final class com.nvidia.spark.rapids.shims.SparkShimImpl { ``` @@ -178,7 +178,7 @@ mvn package -pl dist -am -Dbuildver=340 -DallowConventionalDistJar=true Verify `com.nvidia.spark.rapids.shims.SparkShimImpl` is conventionally loadable: ```bash -$ javap -cp dist/target/rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar com.nvidia.spark.rapids.shims.SparkShimImpl | head -2 +$ javap -cp dist/target/rapids-4-spark_2.12-24.08.0-cuda11.jar com.nvidia.spark.rapids.shims.SparkShimImpl | head -2 Compiled from "SparkShims.scala" public final class com.nvidia.spark.rapids.shims.SparkShimImpl { ``` diff --git a/README.md b/README.md index 297e6eacb2f..af52cc1e358 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ as a `provided` dependency. com.nvidia rapids-4-spark_2.12 - 24.08.0-SNAPSHOT + 24.08.0 provided ``` diff --git a/aggregator/pom.xml b/aggregator/pom.xml index 4f0ea3f6c16..3e5b64604b9 100644 --- a/aggregator/pom.xml +++ b/aggregator/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../jdk-profiles/pom.xml rapids-4-spark-aggregator_2.12 RAPIDS Accelerator for Apache Spark Aggregator Creates an aggregated shaded package of the RAPIDS plugin for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 aggregator diff --git a/api_validation/pom.xml b/api_validation/pom.xml index 7b892754d28..32ee86f53af 100644 --- a/api_validation/pom.xml +++ b/api_validation/pom.xml @@ -22,11 +22,11 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-api-validation_2.12 - 24.08.0-SNAPSHOT + 24.08.0 api_validation diff --git a/datagen/ScaleTest.md b/datagen/ScaleTest.md index 19ca2d21713..f416c52ccd1 100644 --- a/datagen/ScaleTest.md +++ b/datagen/ScaleTest.md @@ -44,7 +44,7 @@ $SPARK_HOME/bin/spark-submit \ --conf spark.sql.parquet.datetimeRebaseModeInWrite=CORRECTED \ --class com.nvidia.rapids.tests.scaletest.ScaleTestDataGen \ # the main class --jars $SPARK_HOME/examples/jars/scopt_2.12-3.7.1.jar \ # one dependency jar just shipped with Spark under $SPARK_HOME -./target/datagen_2.12-24.08.0-SNAPSHOT-spark332.jar \ +./target/datagen_2.12-24.08.0-spark332.jar \ 1 \ 10 \ parquet \ diff --git a/datagen/pom.xml b/datagen/pom.xml index a18797d049f..ba23e4931cb 100644 --- a/datagen/pom.xml +++ b/datagen/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml datagen_2.12 Data Generator Tools for generating large amounts of data - 24.08.0-SNAPSHOT + 24.08.0 datagen diff --git a/delta-lake/delta-20x/pom.xml b/delta-lake/delta-20x/pom.xml index be646b807f5..56695c38fc8 100644 --- a/delta-lake/delta-20x/pom.xml +++ b/delta-lake/delta-20x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-20x_2.12 RAPIDS Accelerator for Apache Spark Delta Lake 2.0.x Support Delta Lake 2.0.x support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-20x diff --git a/delta-lake/delta-21x/pom.xml b/delta-lake/delta-21x/pom.xml index f2915278625..08daaf3510e 100644 --- a/delta-lake/delta-21x/pom.xml +++ b/delta-lake/delta-21x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-21x_2.12 RAPIDS Accelerator for Apache Spark Delta Lake 2.1.x Support Delta Lake 2.1.x support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-21x diff --git a/delta-lake/delta-22x/pom.xml b/delta-lake/delta-22x/pom.xml index 0be1371f4a2..75d8488bdbe 100644 --- a/delta-lake/delta-22x/pom.xml +++ b/delta-lake/delta-22x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-22x_2.12 RAPIDS Accelerator for Apache Spark Delta Lake 2.2.x Support Delta Lake 2.2.x support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-22x diff --git a/delta-lake/delta-23x/pom.xml b/delta-lake/delta-23x/pom.xml index b4bd721dd55..93b909090ae 100644 --- a/delta-lake/delta-23x/pom.xml +++ b/delta-lake/delta-23x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-parent_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../../pom.xml rapids-4-spark-delta-23x_2.12 RAPIDS Accelerator for Apache Spark Delta Lake 2.3.x Support Delta Lake 2.3.x support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-23x diff --git a/delta-lake/delta-24x/pom.xml b/delta-lake/delta-24x/pom.xml index 5573709b446..fac8772f350 100644 --- a/delta-lake/delta-24x/pom.xml +++ b/delta-lake/delta-24x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-24x_2.12 RAPIDS Accelerator for Apache Spark Delta Lake 2.4.x Support Delta Lake 2.4.x support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-24x diff --git a/delta-lake/delta-spark330db/pom.xml b/delta-lake/delta-spark330db/pom.xml index 3ee901eef2c..43c3facea2e 100644 --- a/delta-lake/delta-spark330db/pom.xml +++ b/delta-lake/delta-spark330db/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-spark330db_2.12 RAPIDS Accelerator for Apache Spark Databricks 11.3 Delta Lake Support Databricks 11.3 Delta Lake support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-spark330db diff --git a/delta-lake/delta-spark332db/pom.xml b/delta-lake/delta-spark332db/pom.xml index 92964a59e17..03080a3393b 100644 --- a/delta-lake/delta-spark332db/pom.xml +++ b/delta-lake/delta-spark332db/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-spark332db_2.12 RAPIDS Accelerator for Apache Spark Databricks 12.2 Delta Lake Support Databricks 12.2 Delta Lake support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-spark332db diff --git a/delta-lake/delta-spark341db/pom.xml b/delta-lake/delta-spark341db/pom.xml index 6eaa3fb8c32..58a9771b5af 100644 --- a/delta-lake/delta-spark341db/pom.xml +++ b/delta-lake/delta-spark341db/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-spark341db_2.12 RAPIDS Accelerator for Apache Spark Databricks 13.3 Delta Lake Support Databricks 13.3 Delta Lake support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 false diff --git a/delta-lake/delta-stub/pom.xml b/delta-lake/delta-stub/pom.xml index bfe920e3dc4..102af18a00d 100644 --- a/delta-lake/delta-stub/pom.xml +++ b/delta-lake/delta-stub/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-stub_2.12 RAPIDS Accelerator for Apache Spark Delta Lake Stub Delta Lake stub for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-stub diff --git a/dist/pom.xml b/dist/pom.xml index 814b2bf63db..2e0740c96eb 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../jdk-profiles/pom.xml rapids-4-spark_2.12 RAPIDS Accelerator for Apache Spark Distribution Creates the distribution package of the RAPIDS plugin for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 com.nvidia diff --git a/docs/configs.md b/docs/configs.md index def85c8ac97..4096532d2eb 100644 --- a/docs/configs.md +++ b/docs/configs.md @@ -10,7 +10,7 @@ The following is the list of options that `rapids-plugin-4-spark` supports. On startup use: `--conf [conf key]=[conf value]`. For example: ``` -${SPARK_HOME}/bin/spark-shell --jars rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar \ +${SPARK_HOME}/bin/spark-shell --jars rapids-4-spark_2.12-24.08.0-cuda11.jar \ --conf spark.plugins=com.nvidia.spark.SQLPlugin \ --conf spark.rapids.sql.concurrentGpuTasks=2 ``` diff --git a/integration_tests/README.md b/integration_tests/README.md index 504efb24563..0c69fa7ba56 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -263,7 +263,7 @@ individually, so you don't risk running unit tests along with the integration te http://www.scalatest.org/user_guide/using_the_scalatest_shell ```shell -spark-shell --jars rapids-4-spark-tests_2.12-24.08.0-SNAPSHOT-tests.jar,rapids-4-spark-integration-tests_2.12-24.08.0-SNAPSHOT-tests.jar,scalatest_2.12-3.0.5.jar,scalactic_2.12-3.0.5.jar +spark-shell --jars rapids-4-spark-tests_2.12-24.08.0-tests.jar,rapids-4-spark-integration-tests_2.12-24.08.0-tests.jar,scalatest_2.12-3.0.5.jar,scalactic_2.12-3.0.5.jar ``` First you import the `scalatest_shell` and tell the tests where they can find the test files you @@ -286,7 +286,7 @@ If you just want to verify the SQL replacement is working you will need to add t assumes CUDA 11.0 is being used and the Spark distribution is built with Scala 2.12. ``` -$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar" ./runtests.py +$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-24.08.0-cuda11.jar" ./runtests.py ``` You don't have to enable the plugin for this to work, the test framework will do that for you. @@ -443,7 +443,7 @@ To run cudf_udf tests, need following configuration changes: As an example, here is the `spark-submit` command with the cudf_udf parameter on CUDA 11.0: ``` -$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar,rapids-4-spark-tests_2.12-24.08.0-SNAPSHOT.jar" --conf spark.rapids.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.concurrentPythonWorkers=2 --py-files "rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar" --conf spark.executorEnv.PYTHONPATH="rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar" ./runtests.py --cudf_udf +$SPARK_HOME/bin/spark-submit --jars "rapids-4-spark_2.12-24.08.0-cuda11.jar,rapids-4-spark-tests_2.12-24.08.0.jar" --conf spark.rapids.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.memory.gpu.allocFraction=0.3 --conf spark.rapids.python.concurrentPythonWorkers=2 --py-files "rapids-4-spark_2.12-24.08.0-cuda11.jar" --conf spark.executorEnv.PYTHONPATH="rapids-4-spark_2.12-24.08.0-cuda11.jar" ./runtests.py --cudf_udf ``` ### Enabling fuzz tests diff --git a/integration_tests/ScaleTest.md b/integration_tests/ScaleTest.md index a8f7fb991f3..4bc42280b80 100644 --- a/integration_tests/ScaleTest.md +++ b/integration_tests/ScaleTest.md @@ -97,7 +97,7 @@ $SPARK_HOME/bin/spark-submit \ --conf spark.sql.parquet.datetimeRebaseModeInWrite=CORRECTED \ --jars $SPARK_HOME/examples/jars/scopt_2.12-3.7.1.jar \ --class com.nvidia.spark.rapids.tests.scaletest.ScaleTest \ -./target/rapids-4-spark-integration-tests_2.12-24.08.0-SNAPSHOT-spark332.jar \ +./target/rapids-4-spark-integration-tests_2.12-24.08.0-spark332.jar \ 10 \ 100 \ parquet \ diff --git a/integration_tests/pom.xml b/integration_tests/pom.xml index 75bd244f31d..59bfa1cddb4 100644 --- a/integration_tests/pom.xml +++ b/integration_tests/pom.xml @@ -22,11 +22,11 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-integration-tests_2.12 - 24.08.0-SNAPSHOT + 24.08.0 integration_tests diff --git a/jdk-profiles/pom.xml b/jdk-profiles/pom.xml index 771b5a72476..aa2d9fcfaa5 100644 --- a/jdk-profiles/pom.xml +++ b/jdk-profiles/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent_2.12 - 24.08.0-SNAPSHOT + 24.08.0 com.nvidia rapids-4-spark-jdk-profiles_2.12 pom Shim JDK Profiles - 24.08.0-SNAPSHOT + 24.08.0 jdk9plus diff --git a/jenkins/databricks/create.py b/jenkins/databricks/create.py index 58b583b5c08..f2075917c09 100644 --- a/jenkins/databricks/create.py +++ b/jenkins/databricks/create.py @@ -27,7 +27,7 @@ def main(): workspace = 'https://dbc-9ff9942e-a9c4.cloud.databricks.com' token = '' sshkey = '' - cluster_name = 'CI-GPU-databricks-24.08.0-SNAPSHOT' + cluster_name = 'CI-GPU-databricks-24.08.0' idletime = 240 runtime = '7.0.x-gpu-ml-scala2.12' num_workers = 1 diff --git a/jenkins/version-def.sh b/jenkins/version-def.sh index 8d363fc9d4a..85094a73488 100755 --- a/jenkins/version-def.sh +++ b/jenkins/version-def.sh @@ -27,11 +27,11 @@ done IFS=$PRE_IFS -CUDF_VER=${CUDF_VER:-"24.08.0-SNAPSHOT"} +CUDF_VER=${CUDF_VER:-"24.08.0"} CUDA_CLASSIFIER=${CUDA_CLASSIFIER:-"cuda11"} CLASSIFIER=${CLASSIFIER:-"$CUDA_CLASSIFIER"} # default as CUDA_CLASSIFIER for compatibility -PROJECT_VER=${PROJECT_VER:-"24.08.0-SNAPSHOT"} -PROJECT_TEST_VER=${PROJECT_TEST_VER:-"24.08.0-SNAPSHOT"} +PROJECT_VER=${PROJECT_VER:-"24.08.0"} +PROJECT_TEST_VER=${PROJECT_TEST_VER:-"24.08.0"} SPARK_VER=${SPARK_VER:-"3.2.0"} SPARK_VER_213=${SPARK_VER_213:-"3.3.0"} # Make a best attempt to set the default value for the shuffle shim. diff --git a/pom.xml b/pom.xml index 80cdf968a7d..71e4b496278 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ rapids-4-spark-parent_2.12 RAPIDS Accelerator for Apache Spark Root Project The root project of the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 pom https://nvidia.github.io/spark-rapids/ diff --git a/scala2.13/aggregator/pom.xml b/scala2.13/aggregator/pom.xml index bf643394a98..ced57eaf2dc 100644 --- a/scala2.13/aggregator/pom.xml +++ b/scala2.13/aggregator/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../jdk-profiles/pom.xml rapids-4-spark-aggregator_2.13 RAPIDS Accelerator for Apache Spark Aggregator Creates an aggregated shaded package of the RAPIDS plugin for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 aggregator diff --git a/scala2.13/api_validation/pom.xml b/scala2.13/api_validation/pom.xml index 5cf34d9acf7..9137d17c246 100644 --- a/scala2.13/api_validation/pom.xml +++ b/scala2.13/api_validation/pom.xml @@ -22,11 +22,11 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-api-validation_2.13 - 24.08.0-SNAPSHOT + 24.08.0 api_validation diff --git a/scala2.13/datagen/pom.xml b/scala2.13/datagen/pom.xml index c0195411227..e9aeb875206 100644 --- a/scala2.13/datagen/pom.xml +++ b/scala2.13/datagen/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml datagen_2.13 Data Generator Tools for generating large amounts of data - 24.08.0-SNAPSHOT + 24.08.0 datagen diff --git a/scala2.13/delta-lake/delta-20x/pom.xml b/scala2.13/delta-lake/delta-20x/pom.xml index 40547c3f510..c98cba005c7 100644 --- a/scala2.13/delta-lake/delta-20x/pom.xml +++ b/scala2.13/delta-lake/delta-20x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-20x_2.13 RAPIDS Accelerator for Apache Spark Delta Lake 2.0.x Support Delta Lake 2.0.x support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-20x diff --git a/scala2.13/delta-lake/delta-21x/pom.xml b/scala2.13/delta-lake/delta-21x/pom.xml index 0f228709016..7ceb0a85502 100644 --- a/scala2.13/delta-lake/delta-21x/pom.xml +++ b/scala2.13/delta-lake/delta-21x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-21x_2.13 RAPIDS Accelerator for Apache Spark Delta Lake 2.1.x Support Delta Lake 2.1.x support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-21x diff --git a/scala2.13/delta-lake/delta-22x/pom.xml b/scala2.13/delta-lake/delta-22x/pom.xml index c315b05ad36..fae28b12183 100644 --- a/scala2.13/delta-lake/delta-22x/pom.xml +++ b/scala2.13/delta-lake/delta-22x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-22x_2.13 RAPIDS Accelerator for Apache Spark Delta Lake 2.2.x Support Delta Lake 2.2.x support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-22x diff --git a/scala2.13/delta-lake/delta-23x/pom.xml b/scala2.13/delta-lake/delta-23x/pom.xml index 7ac7c1156dd..d67bdd9cc0b 100644 --- a/scala2.13/delta-lake/delta-23x/pom.xml +++ b/scala2.13/delta-lake/delta-23x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-parent_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../../pom.xml rapids-4-spark-delta-23x_2.13 RAPIDS Accelerator for Apache Spark Delta Lake 2.3.x Support Delta Lake 2.3.x support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-23x diff --git a/scala2.13/delta-lake/delta-24x/pom.xml b/scala2.13/delta-lake/delta-24x/pom.xml index 2d1a1b43e82..ef7332e2bc6 100644 --- a/scala2.13/delta-lake/delta-24x/pom.xml +++ b/scala2.13/delta-lake/delta-24x/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-24x_2.13 RAPIDS Accelerator for Apache Spark Delta Lake 2.4.x Support Delta Lake 2.4.x support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-24x diff --git a/scala2.13/delta-lake/delta-spark330db/pom.xml b/scala2.13/delta-lake/delta-spark330db/pom.xml index 46d6f8798d9..24610e2d1b6 100644 --- a/scala2.13/delta-lake/delta-spark330db/pom.xml +++ b/scala2.13/delta-lake/delta-spark330db/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-spark330db_2.13 RAPIDS Accelerator for Apache Spark Databricks 11.3 Delta Lake Support Databricks 11.3 Delta Lake support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-spark330db diff --git a/scala2.13/delta-lake/delta-spark332db/pom.xml b/scala2.13/delta-lake/delta-spark332db/pom.xml index 93a942e9d44..3961b9c2f38 100644 --- a/scala2.13/delta-lake/delta-spark332db/pom.xml +++ b/scala2.13/delta-lake/delta-spark332db/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-spark332db_2.13 RAPIDS Accelerator for Apache Spark Databricks 12.2 Delta Lake Support Databricks 12.2 Delta Lake support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-spark332db diff --git a/scala2.13/delta-lake/delta-spark341db/pom.xml b/scala2.13/delta-lake/delta-spark341db/pom.xml index 6b5e545291b..7b52aeaf557 100644 --- a/scala2.13/delta-lake/delta-spark341db/pom.xml +++ b/scala2.13/delta-lake/delta-spark341db/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-spark341db_2.13 RAPIDS Accelerator for Apache Spark Databricks 13.3 Delta Lake Support Databricks 13.3 Delta Lake support for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 false diff --git a/scala2.13/delta-lake/delta-stub/pom.xml b/scala2.13/delta-lake/delta-stub/pom.xml index 2ece8c825f6..1aacc8ef6a8 100644 --- a/scala2.13/delta-lake/delta-stub/pom.xml +++ b/scala2.13/delta-lake/delta-stub/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../../jdk-profiles/pom.xml rapids-4-spark-delta-stub_2.13 RAPIDS Accelerator for Apache Spark Delta Lake Stub Delta Lake stub for the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 ../delta-lake/delta-stub diff --git a/scala2.13/dist/pom.xml b/scala2.13/dist/pom.xml index 1e651257f06..b7254c0a449 100644 --- a/scala2.13/dist/pom.xml +++ b/scala2.13/dist/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../jdk-profiles/pom.xml rapids-4-spark_2.13 RAPIDS Accelerator for Apache Spark Distribution Creates the distribution package of the RAPIDS plugin for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 com.nvidia diff --git a/scala2.13/integration_tests/pom.xml b/scala2.13/integration_tests/pom.xml index f7e443d5a22..0eb330e4952 100644 --- a/scala2.13/integration_tests/pom.xml +++ b/scala2.13/integration_tests/pom.xml @@ -22,11 +22,11 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-integration-tests_2.13 - 24.08.0-SNAPSHOT + 24.08.0 integration_tests diff --git a/scala2.13/jdk-profiles/pom.xml b/scala2.13/jdk-profiles/pom.xml index 23fee358668..ed9d9f811c6 100644 --- a/scala2.13/jdk-profiles/pom.xml +++ b/scala2.13/jdk-profiles/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent_2.13 - 24.08.0-SNAPSHOT + 24.08.0 com.nvidia rapids-4-spark-jdk-profiles_2.13 pom Shim JDK Profiles - 24.08.0-SNAPSHOT + 24.08.0 jdk9plus diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index ad353de3602..e15e28fc5b2 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -23,7 +23,7 @@ rapids-4-spark-parent_2.13 RAPIDS Accelerator for Apache Spark Root Project The root project of the RAPIDS Accelerator for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 pom https://nvidia.github.io/spark-rapids/ diff --git a/scala2.13/shim-deps/cloudera/pom.xml b/scala2.13/shim-deps/cloudera/pom.xml index c238ca81af0..8a300857103 100644 --- a/scala2.13/shim-deps/cloudera/pom.xml +++ b/scala2.13/shim-deps/cloudera/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../../pom.xml rapids-4-spark-cdh-bom pom CDH Shim Dependencies - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/cloudera diff --git a/scala2.13/shim-deps/databricks/pom.xml b/scala2.13/shim-deps/databricks/pom.xml index a0459901079..bb31c2b3eb2 100644 --- a/scala2.13/shim-deps/databricks/pom.xml +++ b/scala2.13/shim-deps/databricks/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../../pom.xml rapids-4-spark-db-bom pom Databricks Shim Dependencies - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/databricks diff --git a/scala2.13/shim-deps/pom.xml b/scala2.13/shim-deps/pom.xml index 2db84e7bc31..dce07df5fbc 100644 --- a/scala2.13/shim-deps/pom.xml +++ b/scala2.13/shim-deps/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../jdk-profiles/pom.xml rapids-4-spark-shim-deps-parent_2.13 pom Shim Dependencies Profiles - 24.08.0-SNAPSHOT + 24.08.0 release321cdh diff --git a/scala2.13/shuffle-plugin/pom.xml b/scala2.13/shuffle-plugin/pom.xml index 61e69784972..c5cd9467306 100644 --- a/scala2.13/shuffle-plugin/pom.xml +++ b/scala2.13/shuffle-plugin/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-shuffle_2.13 RAPIDS Accelerator for Apache Spark Shuffle Plugin Accelerated shuffle plugin for the RAPIDS plugin for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 shuffle-plugin diff --git a/scala2.13/sql-plugin-api/pom.xml b/scala2.13/sql-plugin-api/pom.xml index e56caf01867..6da034ac496 100644 --- a/scala2.13/sql-plugin-api/pom.xml +++ b/scala2.13/sql-plugin-api/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-sql-plugin-api_2.13 Module for Non-Shimmable API - 24.08.0-SNAPSHOT + 24.08.0 sql-plugin-api false diff --git a/scala2.13/sql-plugin/pom.xml b/scala2.13/sql-plugin/pom.xml index df3532a3592..28fc2141cc6 100644 --- a/scala2.13/sql-plugin/pom.xml +++ b/scala2.13/sql-plugin/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-sql_2.13 RAPIDS Accelerator for Apache Spark SQL Plugin The RAPIDS SQL plugin for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 sql-plugin diff --git a/scala2.13/tests/pom.xml b/scala2.13/tests/pom.xml index 4a80f1d3942..badd77496ee 100644 --- a/scala2.13/tests/pom.xml +++ b/scala2.13/tests/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-tests_2.13 RAPIDS Accelerator for Apache Spark Tests RAPIDS plugin for Apache Spark integration tests - 24.08.0-SNAPSHOT + 24.08.0 tests diff --git a/scala2.13/tools/pom.xml b/scala2.13/tools/pom.xml index fd14e27c935..badebbaebfd 100644 --- a/scala2.13/tools/pom.xml +++ b/scala2.13/tools/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../jdk-profiles/pom.xml rapids-4-spark-tools-support pom RAPIDS Accelerator for Apache Spark Tools Support Supporting code for RAPIDS Accelerator tools - 24.08.0-SNAPSHOT + 24.08.0 com.nvidia diff --git a/scala2.13/udf-compiler/pom.xml b/scala2.13/udf-compiler/pom.xml index ea16fb9a7c9..d6c29f0b351 100644 --- a/scala2.13/udf-compiler/pom.xml +++ b/scala2.13/udf-compiler/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.13 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-udf_2.13 RAPIDS Accelerator for Apache Spark Scala UDF Plugin The RAPIDS Scala UDF plugin for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 udf-compiler diff --git a/shim-deps/cloudera/pom.xml b/shim-deps/cloudera/pom.xml index e8c66f54755..45ea4b202fc 100644 --- a/shim-deps/cloudera/pom.xml +++ b/shim-deps/cloudera/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../../pom.xml rapids-4-spark-cdh-bom pom CDH Shim Dependencies - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/cloudera diff --git a/shim-deps/databricks/pom.xml b/shim-deps/databricks/pom.xml index 22842b0f7c0..79cca384e82 100644 --- a/shim-deps/databricks/pom.xml +++ b/shim-deps/databricks/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-parent_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../../pom.xml rapids-4-spark-db-bom pom Databricks Shim Dependencies - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/databricks diff --git a/shim-deps/pom.xml b/shim-deps/pom.xml index 5bff137efea..8342fde5752 100644 --- a/shim-deps/pom.xml +++ b/shim-deps/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../jdk-profiles/pom.xml rapids-4-spark-shim-deps-parent_2.12 pom Shim Dependencies Profiles - 24.08.0-SNAPSHOT + 24.08.0 release321cdh diff --git a/shuffle-plugin/pom.xml b/shuffle-plugin/pom.xml index d3498f8b44f..58a5d929845 100644 --- a/shuffle-plugin/pom.xml +++ b/shuffle-plugin/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-shuffle_2.12 RAPIDS Accelerator for Apache Spark Shuffle Plugin Accelerated shuffle plugin for the RAPIDS plugin for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 shuffle-plugin diff --git a/sql-plugin-api/pom.xml b/sql-plugin-api/pom.xml index 5e933899560..3759bd164aa 100644 --- a/sql-plugin-api/pom.xml +++ b/sql-plugin-api/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-sql-plugin-api_2.12 Module for Non-Shimmable API - 24.08.0-SNAPSHOT + 24.08.0 sql-plugin-api false diff --git a/sql-plugin/pom.xml b/sql-plugin/pom.xml index 961e6f08372..5ac084fa4f1 100644 --- a/sql-plugin/pom.xml +++ b/sql-plugin/pom.xml @@ -22,13 +22,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-sql_2.12 RAPIDS Accelerator for Apache Spark SQL Plugin The RAPIDS SQL plugin for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 sql-plugin diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index 4b95ab4b6a6..4f6924458c6 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -2389,7 +2389,7 @@ val SHUFFLE_COMPRESSION_LZ4_CHUNK_SIZE = conf("spark.rapids.shuffle.compression. |On startup use: `--conf [conf key]=[conf value]`. For example: | |``` - |${SPARK_HOME}/bin/spark-shell --jars rapids-4-spark_2.12-24.08.0-SNAPSHOT-cuda11.jar \ + |${SPARK_HOME}/bin/spark-shell --jars rapids-4-spark_2.12-24.08.0-cuda11.jar \ |--conf spark.plugins=com.nvidia.spark.SQLPlugin \ |--conf spark.rapids.sql.concurrentGpuTasks=2 |``` diff --git a/tests/pom.xml b/tests/pom.xml index cff108e3253..d597f8c21c6 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-tests_2.12 RAPIDS Accelerator for Apache Spark Tests RAPIDS plugin for Apache Spark integration tests - 24.08.0-SNAPSHOT + 24.08.0 tests diff --git a/tools/pom.xml b/tools/pom.xml index 5fa7391d34c..5a066571607 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -22,14 +22,14 @@ com.nvidia rapids-4-spark-jdk-profiles_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../jdk-profiles/pom.xml rapids-4-spark-tools-support pom RAPIDS Accelerator for Apache Spark Tools Support Supporting code for RAPIDS Accelerator tools - 24.08.0-SNAPSHOT + 24.08.0 com.nvidia diff --git a/udf-compiler/pom.xml b/udf-compiler/pom.xml index edbdf2148cc..e420354e1ce 100644 --- a/udf-compiler/pom.xml +++ b/udf-compiler/pom.xml @@ -21,13 +21,13 @@ com.nvidia rapids-4-spark-shim-deps-parent_2.12 - 24.08.0-SNAPSHOT + 24.08.0 ../shim-deps/pom.xml rapids-4-spark-udf_2.12 RAPIDS Accelerator for Apache Spark Scala UDF Plugin The RAPIDS Scala UDF plugin for Apache Spark - 24.08.0-SNAPSHOT + 24.08.0 udf-compiler