[WIP] Add support for Cloudera CDS-3.3.2

aggregator/pom.xml

@@ -403,6 +403,29 @@
                 </dependency>
             </dependencies>
         </profile>
+        <profile>
+            <id>release332cdh</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>332cdh</value>
+                </property>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>com.nvidia</groupId>
+                    <artifactId>rapids-4-spark-delta-21x_${scala.binary.version}</artifactId>
+                    <version>${project.version}</version>
+                    <classifier>${spark.version.classifier}</classifier>
+                </dependency>
+                <dependency>
+                    <groupId>com.nvidia</groupId>
+                    <artifactId>rapids-4-spark-delta-22x_${scala.binary.version}</artifactId>
+                    <version>${project.version}</version>
+                    <classifier>${spark.version.classifier}</classifier>
+                </dependency>
+            </dependencies>
+        </profile>
         <profile>
             <id>release330db</id>
             <activation>

datagen/src/main/spark320/scala/org/apache/spark/sql/tests/datagen/datagen/DataGenExprBase.scala

@@ -27,6 +27,7 @@
 {"spark": "330db"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "332db"}
 {"spark": "333"}
 {"spark": "340"}

dist/pom.xml

@@ -133,7 +133,8 @@
                     320,
                     321cdh,
                     330,
-                    330cdh
+                    330cdh,
+                    332cdh
                 </included_buildvers>
             </properties>
         </profile>

integration_tests/pom.xml

@@ -112,6 +112,24 @@
                 </dependency>
             </dependencies>
         </profile>
+        <profile>
+            <id>release332cdh</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>332cdh</value>
+                </property>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>com.nvidia</groupId>
+                    <artifactId>rapids-4-spark-cdh-bom</artifactId>
+                    <version>${project.version}</version>
+                    <type>pom</type>
+                    <scope>provided</scope>
+                </dependency>
+            </dependencies>
+        </profile>
         <profile>
             <!--
                  Note that we are using the Spark version for all of the Databricks dependencies as well.

integration_tests/src/main/python/spark_session.py

@@ -180,8 +180,11 @@ def is_spark_321cdh():
 def is_spark_330cdh():
     return "3.3.0.3.3.718" in spark_version()
 
+def is_spark_332cdh():
+    return "3.3.2.3.3.719" in spark_version()
+
 def is_spark_cdh():
-    return is_spark_321cdh() or is_spark_330cdh()
+    return is_spark_321cdh() or is_spark_330cdh() or is_spark_332cdh()
 
 def is_databricks_version_or_later(major, minor):
     spark = get_spark_i_know_what_i_am_doing()

pom.xml

@@ -458,6 +458,45 @@
                 <module>delta-lake/delta-22x</module>
             </modules>
         </profile>
+        <profile>
+            <id>release332cdh</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>332cdh</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>332cdh</buildver>
+                <spark.version>${spark332cdh.version}</spark.version>
+                <spark.test.version>${spark332cdh.version}</spark.test.version>
+                <parquet.hadoop.version>1.10.99.7.1.9.0-387</parquet.hadoop.version>
+                <iceberg.version>${spark330.iceberg.version}</iceberg.version>
+            </properties>
+            <repositories>
+                <repository>
+                    <id>cloudera-repo</id>
+                    <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
+                    <releases>
+                        <enabled>true</enabled>
+                    </releases>
+                    <snapshots>
+                        <enabled>true</enabled>
+                    </snapshots>
+                </repository>
+            </repositories>
+            <pluginRepositories>
+                <pluginRepository>
+                    <id>cloudera-repo</id>
+                    <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
+                </pluginRepository>
+            </pluginRepositories>
+            <modules>
+                <module>shim-deps/cloudera</module>
+                <module>delta-lake/delta-21x</module>
+                <module>delta-lake/delta-22x</module>
+            </modules>
+        </profile>
         <profile>
             <!-- Note Databricks requires 2 properties -Ddatabricks and -Dbuildver=330db -->
             <!-- Note that 330db backports many features from Spark3.4.0 -->
@@ -654,6 +693,7 @@
         <spark340.version>3.4.0</spark340.version>
         <spark341.version>3.4.1</spark341.version>
         <spark330cdh.version>3.3.0.3.3.7180.0-274</spark330cdh.version>
+        <spark332cdh.version>3.3.2.3.3.7190.0-91</spark332cdh.version>
         <spark330db.version>3.3.0-databricks</spark330db.version>
         <spark332db.version>3.3.2-databricks</spark332db.version>
         <spark350.version>3.5.0</spark350.version>
@@ -699,6 +739,7 @@
             332,
             333,
             330cdh,
+            332cdh,
             340,
             341,
             350

sql-plugin/pom.xml

@@ -139,6 +139,24 @@
                 </dependency>
             </dependencies>
         </profile>
+        <profile>
+            <id>release332cdh</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>332cdh</value>
+                </property>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>com.nvidia</groupId>
+                    <artifactId>rapids-4-spark-cdh-bom</artifactId>
+                    <version>${project.version}</version>
+                    <type>pom</type>
+                    <scope>provided</scope>
+                </dependency>
+            </dependencies>
+        </profile>
         <profile>
             <id>release330db</id>
             <activation>

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AQEUtils.scala

@@ -28,6 +28,7 @@
 {"spark": "330cdh"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala

@@ -28,6 +28,7 @@
 {"spark": "330cdh"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AnsiCastShim.scala

@@ -29,6 +29,7 @@
 {"spark": "330cdh"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/BloomFilterShims.scala

@@ -21,7 +21,7 @@
 {"spark": "320"}
 {"spark": "321"}
 {"spark": "321cdh"}
-{"spark": "321db" }
+{"spark": "321db"}
 {"spark": "322"}
 {"spark": "323"}
 {"spark": "324"}

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CastCheckShims.scala

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+
 /*** spark-rapids-shim-json-lines
 {"spark": "311"}
 {"spark": "312"}
@@ -26,7 +27,6 @@
 {"spark": "323"}
 {"spark": "324"}
 spark-rapids-shim-json-lines ***/
-
 package com.nvidia.spark.rapids.shims
 
 import com.nvidia.spark.rapids.TypeSig

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CastingConfigShim.scala

@@ -29,6 +29,7 @@
 {"spark": "330cdh"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/CreateDataSourceTableAsSelectCommandMetaShims.scala

@@ -30,6 +30,7 @@
 {"spark": "330db"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DecimalArithmeticOverrides.scala

@@ -29,6 +29,7 @@
 {"spark": "330cdh"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DeltaLakeUtils.scala

@@ -28,6 +28,7 @@
 {"spark": "330cdh"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/FileIndexOptionsShims.scala

@@ -30,6 +30,7 @@
 {"spark": "330db"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GetMapValueMeta.scala

@@ -29,6 +29,7 @@
 {"spark": "330cdh"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GlobalLimitShims.scala

@@ -30,6 +30,7 @@
 {"spark": "330db"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "332db"}
 {"spark": "333"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuAggregateInPandasExecMeta.scala

@@ -30,6 +30,7 @@
 {"spark": "330db"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "332db"}
 {"spark": "333"}
 {"spark": "340"}

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuBroadcastJoinMeta.scala

@@ -28,6 +28,7 @@
 {"spark": "330cdh"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuCastShims.scala

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+
 /*** spark-rapids-shim-json-lines
 {"spark": "311"}
 {"spark": "312"}
@@ -29,9 +30,9 @@
 {"spark": "330cdh"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 spark-rapids-shim-json-lines ***/
-
 package com.nvidia.spark.rapids.shims
 
 import ai.rapids.cudf.{ColumnVector, ColumnView}

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuFileFormatDataWriterShim.scala

@@ -30,6 +30,7 @@
 {"spark": "330db"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuOptimizedCreateHiveTableAsSelectCommandShims.scala

@@ -30,6 +30,7 @@
 {"spark": "330db"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuWindowInPandasExec.scala

@@ -28,6 +28,7 @@
 {"spark": "330cdh"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/InSubqueryShims.scala

@@ -21,7 +21,7 @@
 {"spark": "320"}
 {"spark": "321"}
 {"spark": "321cdh"}
-{"spark": "321db" }
+{"spark": "321db"}
 {"spark": "322"}
 {"spark": "323"}
 {"spark": "324"}

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala

@@ -30,6 +30,7 @@
 {"spark": "330db"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "332db"}
 {"spark": "333"}
 {"spark": "340"}

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala

@@ -30,6 +30,7 @@
 {"spark": "330db"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "332db"}
 {"spark": "333"}
 {"spark": "340"}

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/OrcProtoWriterShim.scala

@@ -30,6 +30,7 @@
 {"spark": "330db"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "332db"}
 {"spark": "333"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetLegacyNanoAsLongShims.scala

@@ -28,6 +28,7 @@
 {"spark": "330cdh"}
 {"spark": "330db"}
 {"spark": "331"}
+{"spark": "332cdh"}
 {"spark": "332db"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetStringPredShims.scala

@@ -29,6 +29,7 @@
 {"spark": "330cdh"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "333"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ParquetTimestampNTZShims.scala

@@ -30,6 +30,7 @@
 {"spark": "330db"}
 {"spark": "331"}
 {"spark": "332"}
+{"spark": "332cdh"}
 {"spark": "332db"}
 {"spark": "333"}
 spark-rapids-shim-json-lines ***/