From a1262e1ebde60254fb185c813b22bd0e30115e5c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 5 Oct 2023 16:31:50 -0600 Subject: [PATCH] Only run test_json_ts_formats_round_trip_ntz tests with PySpark 3.4.1 or later Signed-off-by: Andy Grove --- integration_tests/src/main/python/json_test.py | 7 ++++--- integration_tests/src/main/python/spark_session.py | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/integration_tests/src/main/python/json_test.py b/integration_tests/src/main/python/json_test.py index 181e8f11bd2..d866de6a04b 100644 --- a/integration_tests/src/main/python/json_test.py +++ b/integration_tests/src/main/python/json_test.py @@ -21,7 +21,8 @@ from datetime import timezone from conftest import is_databricks_runtime from marks import approximate_float, allow_non_gpu, ignore_order -from spark_session import with_cpu_session, with_gpu_session, is_before_spark_330 +from spark_session import with_cpu_session, with_gpu_session, is_before_spark_330, is_before_spark_340, \ + is_before_spark_341 json_supported_gens = [ # Spark does not escape '\r' or '\n' even though it uses it to mark end of record @@ -203,7 +204,7 @@ def test_json_ts_formats_round_trip(spark_tmp_path, date_format, ts_part, v1_ena conf=updated_conf) @allow_non_gpu('FileSourceScanExec', 'ProjectExec') -@pytest.mark.skipif(is_before_spark_340(), reason='`TIMESTAMP_NTZ` is only supported in Spark 340+') +@pytest.mark.skipif(is_before_spark_341(), reason='`TIMESTAMP_NTZ` is only supported in PySpark 341+') @pytest.mark.parametrize('ts_part', json_supported_ts_parts) @pytest.mark.parametrize('date_format', json_supported_date_formats) @pytest.mark.parametrize("timestamp_type", ["TIMESTAMP_LTZ", "TIMESTAMP_NTZ"]) @@ -211,7 +212,7 @@ def test_json_ts_formats_round_trip_ntz_v1(spark_tmp_path, date_format, ts_part, json_ts_formats_round_trip_ntz(spark_tmp_path, date_format, ts_part, timestamp_type, 'json', 'FileSourceScanExec') @allow_non_gpu('BatchScanExec', 'ProjectExec') -@pytest.mark.skipif(is_before_spark_340(), reason='`TIMESTAMP_NTZ` is only supported in Spark 340+') +@pytest.mark.skipif(is_before_spark_341(), reason='`TIMESTAMP_NTZ` is only supported in PySpark 341+') @pytest.mark.parametrize('ts_part', json_supported_ts_parts) @pytest.mark.parametrize('date_format', json_supported_date_formats) @pytest.mark.parametrize("timestamp_type", ["TIMESTAMP_LTZ", "TIMESTAMP_NTZ"]) diff --git a/integration_tests/src/main/python/spark_session.py b/integration_tests/src/main/python/spark_session.py index bbc8c2cc414..cba36dcbe0e 100644 --- a/integration_tests/src/main/python/spark_session.py +++ b/integration_tests/src/main/python/spark_session.py @@ -153,6 +153,9 @@ def is_before_spark_331(): def is_before_spark_340(): return spark_version() < "3.4.0" +def is_before_spark_341(): + return spark_version() < "3.4.1" + def is_before_spark_350(): return spark_version() < "3.5.0"