From 384b2a95c7fead007136f737624ccb77ec95c0dd Mon Sep 17 00:00:00 2001 From: Chong Gao Date: Fri, 29 Dec 2023 15:35:22 +0800 Subject: [PATCH] Fix from_utc_timestamp failure on Cloudera when TZ is Iran (#10119) Signed-off-by: Chong Gao --- integration_tests/README.md | 4 ++-- integration_tests/src/main/python/date_time_test.py | 2 +- .../com/nvidia/spark/rapids/timezone/TimeZonePerfSuite.scala | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/integration_tests/README.md b/integration_tests/README.md index 9e4ba5a378e..01bf382ca41 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -350,10 +350,10 @@ But only a small amout of cases are verifing with non-UTC time zone in the pre-m When adding cases, should also check non-UTC is working besides the default UTC time zone. Please test the following time zones: ```shell -$ TZ=Iran ./integration_tests/run_pyspark_from_build.sh +$ TZ=Asia/Shanghai ./integration_tests/run_pyspark_from_build.sh $ TZ=America/Los_Angeles ./integration_tests/run_pyspark_from_build.sh ``` -`Iran` is non-DST(Daylight Savings Time) time zone and `America/Los_Angeles` is DST time zone. +`Asia/Shanghai` is non-DST(Daylight Savings Time) time zone and `America/Los_Angeles` is DST time zone. If the new added cases failed with non-UTC, then should allow the operator(does not support non-UTC) fallback, For example, add the following annotation to the case: diff --git a/integration_tests/src/main/python/date_time_test.py b/integration_tests/src/main/python/date_time_test.py index b7b95a89b5c..ca62a5a75e0 100644 --- a/integration_tests/src/main/python/date_time_test.py +++ b/integration_tests/src/main/python/date_time_test.py @@ -287,7 +287,7 @@ def test_unsupported_fallback_to_unix_timestamp(data_gen): "to_unix_timestamp(a, b)"), "ToUnixTimestamp") -@pytest.mark.parametrize('time_zone', ["Asia/Shanghai", "Iran", "UTC", "UTC+0", "UTC-0", "GMT", "GMT+0", "GMT-0"], ids=idfn) +@pytest.mark.parametrize('time_zone', ["Asia/Shanghai", "UTC", "UTC+0", "UTC-0", "GMT", "GMT+0", "GMT-0"], ids=idfn) @pytest.mark.parametrize('data_gen', [timestamp_gen], ids=idfn) @tz_sensitive_test @allow_non_gpu(*non_utc_allow) diff --git a/tests/src/test/scala/com/nvidia/spark/rapids/timezone/TimeZonePerfSuite.scala b/tests/src/test/scala/com/nvidia/spark/rapids/timezone/TimeZonePerfSuite.scala index 49ee394e904..e61ab512d1f 100644 --- a/tests/src/test/scala/com/nvidia/spark/rapids/timezone/TimeZonePerfSuite.scala +++ b/tests/src/test/scala/com/nvidia/spark/rapids/timezone/TimeZonePerfSuite.scala @@ -34,7 +34,7 @@ import org.apache.spark.sql.types._ * A simple test performance framework for non-UTC features. * Usage: * - * argLine="-DTZs=Iran,Asia/Shanghai -DenableTimeZonePerf=true" \ + * argLine="-DTZs=Asia/Shanghai,Japan -DenableTimeZonePerf=true" \ * mvn test -Dbuildver=311 -DwildcardSuites=com.nvidia.spark.rapids.timezone.TimeZonePerfSuite * Note: * Generate a Parquet file with 6 columns: @@ -51,7 +51,7 @@ import org.apache.spark.sql.types._ class TimeZonePerfSuite extends SparkQueryCompareTestSuite with BeforeAndAfterAll { // perf test is disabled by default since it's a long running time in UT. private val enablePerfTest = java.lang.Boolean.getBoolean("enableTimeZonePerf") - private val timeZoneStrings = System.getProperty("TZs", "Iran") + private val timeZoneStrings = System.getProperty("TZs", "Asia/Shanghai") // rows for perf test private val numRows: Long = 1024L * 1024L * 10L