From 3eac082b6315b75ca910d16fdc1b80e351901b40 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 28 Nov 2023 12:00:30 -0700 Subject: [PATCH 1/3] Add support for decimal in to_json --- integration_tests/src/main/python/json_test.py | 3 +++ .../src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala | 2 ++ 2 files changed, 5 insertions(+) diff --git a/integration_tests/src/main/python/json_test.py b/integration_tests/src/main/python/json_test.py index 41571a203d5..93ec4e99a20 100644 --- a/integration_tests/src/main/python/json_test.py +++ b/integration_tests/src/main/python/json_test.py @@ -782,6 +782,9 @@ def test_read_case_col_name(spark_tmp_path, v1_enabled_list, col_name): short_gen, int_gen, long_gen, + decimal_gen_32bit, + decimal_gen_64bit, + decimal_gen_128bit, pytest.param(float_gen, marks=pytest.mark.xfail(reason='https://github.com/NVIDIA/spark-rapids/issues/9350')), pytest.param(double_gen, marks=pytest.mark.xfail(reason='https://github.com/NVIDIA/spark-rapids/issues/9350')), pytest.param(date_gen, marks=pytest.mark.xfail(reason='https://github.com/NVIDIA/spark-rapids/issues/9515')), diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 8119e78d988..825b7c4a2df 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -3600,8 +3600,10 @@ object GpuOverrides extends Logging { TypeSig.STRING, Seq(ParamCheck("struct", (TypeSig.BOOLEAN + TypeSig.STRING + TypeSig.integral + TypeSig.FLOAT + + TypeSig.DECIMAL_64 + TypeSig.DECIMAL_128 + TypeSig.DOUBLE + TypeSig.STRUCT + TypeSig.ARRAY + TypeSig.MAP).nested(), (TypeSig.BOOLEAN + TypeSig.STRING + TypeSig.integral + TypeSig.FLOAT + + TypeSig.DECIMAL_64 + TypeSig.DECIMAL_128 + TypeSig.DOUBLE + TypeSig.STRUCT + TypeSig.ARRAY + TypeSig.MAP).nested() ))), (a, conf, p, r) => new UnaryExprMeta[StructsToJson](a, conf, p, r) { From d0ed1f177ab995e50ce4d07449b56178a17854a5 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 28 Nov 2023 12:10:01 -0700 Subject: [PATCH 2/3] update generated docs Signed-off-by: Andy Grove --- docs/supported_ops.md | 2 +- tools/generated_files/supportedExprs.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/supported_ops.md b/docs/supported_ops.md index 1566a291f36..af5ed1a480e 100644 --- a/docs/supported_ops.md +++ b/docs/supported_ops.md @@ -14544,7 +14544,7 @@ are limited. S - +S diff --git a/tools/generated_files/supportedExprs.csv b/tools/generated_files/supportedExprs.csv index 14b2f8f784d..f978ac81424 100644 --- a/tools/generated_files/supportedExprs.csv +++ b/tools/generated_files/supportedExprs.csv @@ -539,7 +539,7 @@ StringTrimLeft,S,`ltrim`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA, StringTrimRight,S,`rtrim`,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA StringTrimRight,S,`rtrim`,None,project,trimStr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA StringTrimRight,S,`rtrim`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -StructsToJson,NS,`to_json`,This is disabled by default because to_json support is experimental. See compatibility guide for more information.,project,struct,S,S,S,S,S,S,S,NA,NA,S,NA,NA,NA,NA,S,S,S,NA +StructsToJson,NS,`to_json`,This is disabled by default because to_json support is experimental. See compatibility guide for more information.,project,struct,S,S,S,S,S,S,S,NA,NA,S,S,NA,NA,NA,S,S,S,NA StructsToJson,NS,`to_json`,This is disabled by default because to_json support is experimental. See compatibility guide for more information.,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA Substring,S,`substr`; `substring`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA Substring,S,`substr`; `substring`,None,project,pos,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA From fce51877cf2c0643a2752545724169d10d8d2833 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 29 Nov 2023 09:00:05 -0700 Subject: [PATCH 3/3] address feedback --- .../src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 825b7c4a2df..34f8134a0b8 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -3600,10 +3600,10 @@ object GpuOverrides extends Logging { TypeSig.STRING, Seq(ParamCheck("struct", (TypeSig.BOOLEAN + TypeSig.STRING + TypeSig.integral + TypeSig.FLOAT + - TypeSig.DECIMAL_64 + TypeSig.DECIMAL_128 + + TypeSig.DECIMAL_128 + TypeSig.DOUBLE + TypeSig.STRUCT + TypeSig.ARRAY + TypeSig.MAP).nested(), (TypeSig.BOOLEAN + TypeSig.STRING + TypeSig.integral + TypeSig.FLOAT + - TypeSig.DECIMAL_64 + TypeSig.DECIMAL_128 + + TypeSig.DECIMAL_128 + TypeSig.DOUBLE + TypeSig.STRUCT + TypeSig.ARRAY + TypeSig.MAP).nested() ))), (a, conf, p, r) => new UnaryExprMeta[StructsToJson](a, conf, p, r) {