Fix the mismatching default configs in integration tests (#11283)

* Add a new interface to retrieve all configs with their defaults; Add a new stage for integration test to populate default configs Signed-off-by: Jihoon Son <[email protected]> * address comments * missing version update in 2.13 pom * fix match arms * take the json file path as an input * add the new config file in the assembly * missing 2.13 change * use maven build directory var * revert unintended change * remove unnecessary clean * Add a new interface to retrieve all configs with their defaults; Add a new stage for integration test to populate default configs Signed-off-by: Jihoon Son <[email protected]> * address comments * missing version update in 2.13 pom * fix match arms * take the json file path as an input * add the new config file in the assembly * missing 2.13 change * use maven build directory var * revert unintended change * remove unnecessary clean * Add things in RapidsConf * missing change for 2.13 * fix directory path for scala 2.13 * exclude jackson from spark-hive * missing change for 2.13 * exclude old jackson stuff from iceberg * copyrights * antrun * fix config file path * move most dump changes to rapids conf - fork generation step with maven.compile.classpath - change to a phase before package Signed-off-by: Gera Shegalov <[email protected]> * clean up after merge * scala 2.13 * more strict arg check * unpack ambiguous string arguments * allow legacy negative scale for decimals for some tests * should fork for RapidsConf * remove System.exit() from RapidsConf.main() * missing change for scala 2.13 * Fix more tests to set configs * add back explicit configs --------- Signed-off-by: Jihoon Son <[email protected]> Signed-off-by: Gera Shegalov <[email protected]> Co-authored-by: Gera Shegalov <[email protected]>
NVIDIA · Aug 15, 2024 · bc8c577 · bc8c577
1 parent 25be396
commit bc8c577
Show file tree

Hide file tree

Showing 17 changed files with 343 additions and 85 deletions.
diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2023, NVIDIA CORPORATION.
+# Copyright (c) 2020-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -62,3 +62,6 @@ def pytest_addoption(parser):
     parser.addoption(
         "--pyarrow_test", action='store_true', default=False, help="if enable pyarrow tests"
     )
+    parser.addoption(
+        "--default_configs_path", action="store", default=None, help="path to a JSON file that stores default configs for integration test"
+    )
diff --git a/integration_tests/pom.xml b/integration_tests/pom.xml
@@ -68,6 +68,16 @@
             <!-- for hive udf test cases -->
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-hive_${scala.binary.version}</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <!-- https://mvnrepository.com/artifact/com.github.scopt/scopt -->
         <dependency>
@@ -78,6 +88,31 @@
     </dependencies>
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>populate-default-configs-for-testing</id>
+                        <phase>generate-test-resources</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <target>
+                                <taskdef resource="net/sf/antcontrib/antcontrib.properties"/>
+                                <java classname="com.nvidia.spark.rapids.tests.DumpDefaultConfigs"
+                                    failonerror="true"
+                                    classpathref="maven.compile.classpath"
+                                    fork="true">
+                                    <arg value="json"/>
+                                    <arg value="${project.build.directory}/spark-rapids-default-configs.json"/>
+                                </java>
+                            </target>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <artifactId>maven-assembly-plugin</artifactId>
                 <version>3.6.0</version>
@@ -181,7 +216,7 @@
                 <artifactId>exec-maven-plugin</artifactId>
                 <executions>
                     <execution>
-                        <id>run pyspark tests</id>
+                        <id>run-pyspark-tests</id>
                         <phase>verify</phase><!--run after packageing and collecting dependencies-->
                         <goals>
                             <goal>exec</goal>

diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh
@@ -223,6 +223,7 @@ else
 
     REPORT_CHARS=${REPORT_CHARS:="fE"} # default as (f)ailed, (E)rror
     STD_INPUT_PATH="$INPUT_PATH"/src/test/resources
+    DEFAULT_CONFIGS_PATH=${DEFAULT_CONFIGS_PATH:-${TARGET_DIR}/spark-rapids-default-configs.json}
     TEST_COMMON_OPTS=(-v
           -r"$REPORT_CHARS"
           "$TEST_TAGS"
@@ -232,6 +233,7 @@ else
           "$TEST_ARGS"
           $RUN_TEST_PARAMS
           --junitxml=TEST-pytest-`date +%s%N`.xml
+          --default_configs_path="${DEFAULT_CONFIGS_PATH}"
           "$@")
 
     NUM_LOCAL_EXECS=${NUM_LOCAL_EXECS:-0}

diff --git a/integration_tests/src/assembly/bin.xml b/integration_tests/src/assembly/bin.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020-2023, NVIDIA CORPORATION.
+  Copyright (c) 2020-2024, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -50,6 +50,10 @@
             <source>${project.build.outputDirectory}/rapids4spark-version-info.properties</source>
             <outputDirectory>integration_tests</outputDirectory>
         </file>
+        <file>
+            <source>${project.build.directory}/spark-rapids-default-configs.json</source>
+            <outputDirectory>integration_tests</outputDirectory>
+        </file>
     </files>
     <fileSets>
         <fileSet>