before java 17

linkedin · Oct 3, 2024 · afe2de8 · afe2de8
1 parent 93e357e
commit afe2de8
Show file tree

Hide file tree

Showing 19 changed files with 48 additions and 37 deletions.
diff --git a/apps/spark/src/main/java/com/linkedin/openhouse/jobs/exception/TableValidationException.java b/apps/spark/src/main/java/com/linkedin/openhouse/jobs/exception/TableValidationException.java
@@ -8,5 +8,5 @@ public TableValidationException(String message) {
 
   public TableValidationException(String message, Throwable cause) {
     super(message, cause);
-  };
+  }
 }
diff --git a/build.gradle b/build.gradle
@@ -58,15 +58,15 @@ allprojects {
   configurations.all {
     resolutionStrategy {
       force 'com.fasterxml.jackson:jackson-bom:2.13.4'
-      force 'com.fasterxml.jackson.core:jackson-databind:2.13.4'
+      force 'com.fasterxml.jackson.core:jackson-databind:2.14.2'
       force 'org.apache.orc:orc-core:1.8.3'
       force 'com.google.guava:guava:31.1-jre'
     }
   }
 
   plugins.withType(JavaPlugin) {
     dependencies {
-
+//      implementation "javax.annotation:javax.annotation-api:1.3.2"
       testImplementation "org.assertj:assertj-core:3.24.2" //assertions library
       testImplementation "org.junit.jupiter:junit-jupiter-api:5.10.0"
       testImplementation "org.junit.jupiter:junit-jupiter-params:5.10.0"

diff --git a/integrations/java-iceberg-1.5/openhouse-java-itest/build.gradle b/integrations/java-iceberg-1.5/openhouse-java-itest/build.gradle
@@ -4,7 +4,7 @@ plugins {
 }
 
 ext {
-  icebergVersion = '1.2.0'
+  icebergVersion = '1.5.2'
 }
 dependencies {
   testImplementation(project(path: ':integrations:java:openhouse-java-runtime', configuration: 'shadow'))

diff --git a/integrations/java-iceberg-1.5/openhouse-java-runtime/build.gradle b/integrations/java-iceberg-1.5/openhouse-java-runtime/build.gradle
@@ -28,7 +28,7 @@ ext {
 dependencies {
   compileOnly project(':client:secureclient')
   compileOnly project(':client:tableclient')
-  compileOnly("org.apache.iceberg:iceberg-spark-runtime-3.1_2.12:" + icebergVersion)
+  compileOnly("org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:" + icebergVersion)
   compileOnly ("org.springframework.boot:spring-boot-starter-webflux:" + springVersion)
 
   implementation 'org.apache.commons:commons-lang3:3.12.0'

diff --git a/integrations/spark-3.5/openhouse-spark-itest/build.gradle b/integrations/spark-3.5/openhouse-spark-itest/build.gradle
@@ -3,15 +3,19 @@ plugins {
   id 'openhouse.maven-publish'
 }
 
+ext {
+  sparkVersion = '3.5.2'
+}
+
 dependencies {
 
   testImplementation 'com.google.code.gson:gson:2.8.9'
-  
-  testImplementation(project(path: ':integrations:spark:openhouse-spark-runtime_2.12', configuration: 'shadow')) {
+
+  testImplementation(project(path: ':integrations:spark-3.5:openhouse-spark-3.5-runtime_2.12', configuration: 'shadow')) {
     exclude group: 'org.apache.commons', module: 'commons-lang3'
   }
 
-  testImplementation("org.apache.spark:spark-sql_2.12:" + spark_version){
+  testImplementation("org.apache.spark:spark-sql_2.12:" + sparkVersion){
     // These classes are available from `client-codegen-convention.gradle`
     exclude group: "io.netty"
   }
@@ -22,8 +26,8 @@ dependencies {
   testImplementation 'org.apache.commons:commons-lang3:3.12.0'
   testImplementation "com.squareup.okhttp3:okhttp:4.9.3"
   testImplementation "com.squareup.okhttp3:mockwebserver:4.9.3"
-  // Otherwise throws the error: Scala module 2.10.0 requires Jackson Databind version >= 2.10.0 and < 2.11.0
-  testImplementation "com.fasterxml.jackson.module:jackson-module-scala_2.12:2.13.1"
+  // Otherwise throws the error: Scala module 2.15.2 requires Jackson Databind version >= 2.15.0 and < 2.16.0
+//  testImplementation "com.fasterxml.jackson.module:jackson-module-scala_2.12:2.15.2"
   testImplementation "org.openapitools:jackson-databind-nullable:0.2.1"
   testImplementation "io.netty:netty-resolver-dns-native-macos:4.1.70.Final:osx-x86_64"
 }

diff --git a/integrations/spark-3.5/openhouse-spark-runtime/build.gradle b/integrations/spark-3.5/openhouse-spark-runtime/build.gradle
@@ -7,8 +7,8 @@ plugins {
 }
 
 ext {
-  icebergVersion = '1.2.0'
-  sparkVersion = '3.1.1'
+  icebergVersion = '1.5.2'
+  sparkVersion = '3.5.2'
 }
 
 configurations {
@@ -36,7 +36,7 @@ dependencies {
   runtimeOnly "org.antlr:antlr4-runtime:4.7.1"
   antlr "org.antlr:antlr4:4.7.1"
 
-  compileOnly(project(path: ':integrations:java:openhouse-java-runtime', configuration: 'shadow'))
+  compileOnly(project(path: ':integrations:java-iceberg-1.5:openhouse-java-iceberg-1.5-runtime', configuration: 'shadow'))
   compileOnly("org.apache.spark:spark-hive_2.12:${sparkVersion}") {
     exclude group: 'org.apache.avro', module: 'avro'
     exclude group: 'org.apache.arrow'
@@ -46,21 +46,21 @@ dependencies {
     exclude group: 'org.apache.hadoop', module: 'hadoop-client'
   }
 
-  testImplementation("org.apache.iceberg:iceberg-spark-runtime-3.1_2.12:" + icebergVersion) {
+  testImplementation("org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:" + icebergVersion) {
     exclude group: "io.netty"
   }
   testImplementation(project(':tables-test-fixtures_2.12')) {
     exclude group: "io.netty"
   }
-  testImplementation('org.apache.spark:spark-sql_2.12:' + spark_version){
+  testImplementation('org.apache.spark:spark-sql_2.12:' + sparkVersion){
     // These classes are available from `client-codegen-convention.gradle`
   }
   testImplementation(project(path: ':integrations:java:openhouse-java-runtime', configuration: 'shadow'))
 
   fatJarPackagedDependencies(project(path: ':integrations:java:openhouse-java-runtime', configuration: 'shadow')) {
     transitive = false
   }
-  implementation("org.apache.iceberg:iceberg-spark-runtime-3.1_2.12:" + icebergVersion)
+  implementation("org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:" + icebergVersion)
 
   generateGrammarSource {
     maxHeapSize = "64m"

diff --git a/...in/openhouse/spark/sql/catalyst/parser/extensions/OpenhouseSparkSqlExtensionsParser.scala b/...in/openhouse/spark/sql/catalyst/parser/extensions/OpenhouseSparkSqlExtensionsParser.scala
@@ -89,6 +89,9 @@ class OpenhouseSparkSqlExtensionsParser (delegate: ParserInterface) extends Pars
     toResult(parser)
   }
 
+  override def parseQuery(sqlText: String): LogicalPlan = {
+    parsePlan(sqlText)
+  }
 }
 
 /* Copied from Apache Spark's to avoid dependency on Spark Internals */

diff --git a/.../scala/com/linkedin/openhouse/spark/sql/catalyst/plans/logical/GrantRevokeStatement.scala b/.../scala/com/linkedin/openhouse/spark/sql/catalyst/plans/logical/GrantRevokeStatement.scala
@@ -1,9 +1,9 @@
 package com.linkedin.openhouse.spark.sql.catalyst.plans.logical
 
 import com.linkedin.openhouse.spark.sql.catalyst.enums.GrantableResourceTypes.GrantableResourceType
-import org.apache.spark.sql.catalyst.plans.logical.Command
+import org.apache.spark.sql.catalyst.plans.logical.LeafCommand
 
-case class GrantRevokeStatement(isGrant: Boolean, resourceType: GrantableResourceType, resourceName: Seq[String], privilege: String, principal: String) extends Command {
+case class GrantRevokeStatement(isGrant: Boolean, resourceType: GrantableResourceType, resourceName: Seq[String], privilege: String, principal: String) extends LeafCommand {
   override def simpleString(maxFields: Int): String = {
     s"GrantRevokeStatement: isGrant ${isGrant}, ${resourceType} ${resourceName} ${privilege} ${principal}"
   }

diff --git a/...in/scala/com/linkedin/openhouse/spark/sql/catalyst/plans/logical/SetColumnPolicyTag.scala b/...in/scala/com/linkedin/openhouse/spark/sql/catalyst/plans/logical/SetColumnPolicyTag.scala
@@ -1,8 +1,8 @@
 package com.linkedin.openhouse.spark.sql.catalyst.plans.logical
 
-import org.apache.spark.sql.catalyst.plans.logical.Command
+import org.apache.spark.sql.catalyst.plans.logical.LeafCommand
 
-case class SetColumnPolicyTag(tableName: Seq[String], colName: String, policyTags: Seq[String]) extends Command {
+case class SetColumnPolicyTag(tableName: Seq[String], colName: String, policyTags: Seq[String]) extends LeafCommand {
   override def simpleString(maxFields: Int): String = {
     s"SetColumnPolicyTag: ${tableName} ${colName} ${policyTags}"
   }

diff --git a/...in/scala/com/linkedin/openhouse/spark/sql/catalyst/plans/logical/SetRetentionPolicy.scala b/...in/scala/com/linkedin/openhouse/spark/sql/catalyst/plans/logical/SetRetentionPolicy.scala
@@ -1,8 +1,8 @@
 package com.linkedin.openhouse.spark.sql.catalyst.plans.logical
 
-import org.apache.spark.sql.catalyst.plans.logical.Command
+import org.apache.spark.sql.catalyst.plans.logical.LeafCommand
 
-case class SetRetentionPolicy(tableName: Seq[String], granularity: String, count: Int, colName: Option[String], colPattern: Option[String]) extends Command {
+case class SetRetentionPolicy(tableName: Seq[String], granularity: String, count: Int, colName: Option[String], colPattern: Option[String]) extends LeafCommand {
   override def simpleString(maxFields: Int): String = {
     s"SetRetentionPolicy: ${tableName} ${count} ${granularity} ${colName.getOrElse("")} ${colPattern.getOrElse("")}"
   }

diff --git a/...main/scala/com/linkedin/openhouse/spark/sql/catalyst/plans/logical/SetSharingPolicy.scala b/...main/scala/com/linkedin/openhouse/spark/sql/catalyst/plans/logical/SetSharingPolicy.scala
@@ -1,8 +1,8 @@
 package com.linkedin.openhouse.spark.sql.catalyst.plans.logical
 
-import org.apache.spark.sql.catalyst.plans.logical.Command
+import org.apache.spark.sql.catalyst.plans.logical.LeafCommand
 
-case class SetSharingPolicy(tableName: Seq[String], sharing: String) extends Command {
+case class SetSharingPolicy(tableName: Seq[String], sharing: String) extends LeafCommand {
   override def simpleString(maxFields: Int): String = {
     s"SetSharingPolicy: ${tableName} ${sharing}"
   }

diff --git a/...n/scala/com/linkedin/openhouse/spark/sql/catalyst/plans/logical/ShowGrantsStatement.scala b/...n/scala/com/linkedin/openhouse/spark/sql/catalyst/plans/logical/ShowGrantsStatement.scala
@@ -2,11 +2,11 @@ package com.linkedin.openhouse.spark.sql.catalyst.plans.logical
 
 import com.linkedin.openhouse.spark.sql.catalyst.enums.GrantableResourceTypes.GrantableResourceType
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
-import org.apache.spark.sql.catalyst.plans.logical.Command
+import org.apache.spark.sql.catalyst.plans.logical.LeafCommand
 import org.apache.spark.sql.types.StringType
 
 
-case class ShowGrantsStatement(resourceType: GrantableResourceType, resourceName: Seq[String]) extends Command {
+case class ShowGrantsStatement(resourceType: GrantableResourceType, resourceName: Seq[String]) extends LeafCommand {
 
   override lazy val output: Seq[Attribute] = Seq(
     AttributeReference("privilege", StringType, nullable = false)(),

diff --git a/.../com/linkedin/openhouse/spark/sql/execution/datasources/v2/GrantRevokeStatementExec.scala b/.../com/linkedin/openhouse/spark/sql/execution/datasources/v2/GrantRevokeStatementExec.scala
@@ -10,15 +10,15 @@ import org.apache.iceberg.spark.Spark3Util
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
-import org.apache.spark.sql.execution.datasources.v2.V2CommandExec
+import org.apache.spark.sql.execution.datasources.v2.LeafV2CommandExec
 
 case class GrantRevokeStatementExec(
   isGrant: Boolean,
   resourceType: GrantableResourceType,
   catalog: TableCatalog,
   ident: Identifier,
   privilege: String,
-  principal: String) extends V2CommandExec {
+  principal: String) extends LeafV2CommandExec {
 
   override lazy val output: Seq[Attribute] = Nil
 

diff --git a/...la/com/linkedin/openhouse/spark/sql/execution/datasources/v2/SetColumnPolicyTagExec.scala b/...la/com/linkedin/openhouse/spark/sql/execution/datasources/v2/SetColumnPolicyTagExec.scala
@@ -4,13 +4,13 @@ import org.apache.iceberg.spark.source.SparkTable
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
-import org.apache.spark.sql.execution.datasources.v2.V2CommandExec
+import org.apache.spark.sql.execution.datasources.v2.LeafV2CommandExec
 
 case class SetColumnPolicyTagExec(
   catalog: TableCatalog,
   ident: Identifier,
   colName: String,
-  policyTags: Seq[String]) extends V2CommandExec {
+  policyTags: Seq[String]) extends LeafV2CommandExec {
 
   override lazy val output: Seq[Attribute] = Nil
 

diff --git a/...la/com/linkedin/openhouse/spark/sql/execution/datasources/v2/SetRetentionPolicyExec.scala b/...la/com/linkedin/openhouse/spark/sql/execution/datasources/v2/SetRetentionPolicyExec.scala
@@ -4,7 +4,7 @@ import org.apache.iceberg.spark.source.SparkTable
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
-import org.apache.spark.sql.execution.datasources.v2.V2CommandExec
+import org.apache.spark.sql.execution.datasources.v2.LeafV2CommandExec
 
 case class SetRetentionPolicyExec(
   catalog: TableCatalog,
@@ -13,7 +13,7 @@ case class SetRetentionPolicyExec(
   count: Int,
   colName: Option[String],
   colPattern: Option[String]
-                                 ) extends V2CommandExec {
+                                 ) extends LeafV2CommandExec {
 
   override lazy val output: Seq[Attribute] = Nil
 

diff --git a/...cala/com/linkedin/openhouse/spark/sql/execution/datasources/v2/SetSharingPolicyExec.scala b/...cala/com/linkedin/openhouse/spark/sql/execution/datasources/v2/SetSharingPolicyExec.scala
@@ -4,12 +4,12 @@ import org.apache.iceberg.spark.source.SparkTable
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
-import org.apache.spark.sql.execution.datasources.v2.V2CommandExec
+import org.apache.spark.sql.execution.datasources.v2.LeafV2CommandExec
 
 case class SetSharingPolicyExec(
   catalog: TableCatalog,
   ident: Identifier,
-  sharing: String) extends V2CommandExec {
+  sharing: String) extends LeafV2CommandExec {
 
   override lazy val output: Seq[Attribute] = Nil
 

diff --git a/...a/com/linkedin/openhouse/spark/sql/execution/datasources/v2/ShowGrantsStatementExec.scala b/...a/com/linkedin/openhouse/spark/sql/execution/datasources/v2/ShowGrantsStatementExec.scala
@@ -9,7 +9,7 @@ import org.apache.iceberg.spark.Spark3Util
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericInternalRow}
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
-import org.apache.spark.sql.execution.datasources.v2.V2CommandExec
+import org.apache.spark.sql.execution.datasources.v2.LeafV2CommandExec
 import org.apache.spark.sql.execution.LeafExecNode
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -20,7 +20,7 @@ case class ShowGrantsStatementExec(
   output: Seq[Attribute],
   resourceType: GrantableResourceType,
   catalog: TableCatalog,
-  ident: Identifier) extends V2CommandExec  with LeafExecNode {
+  ident: Identifier) extends LeafV2CommandExec with LeafExecNode {
 
   override protected def run(): Seq[InternalRow] = {
     /** Extract {@link OpenHouseCatalog}  from {@link TableCatalog} */

diff --git a/...ations/spark/src/main/java/com/linkedin/openhouse/spark/builder/PartitionSpecBuilder.java b/...ations/spark/src/main/java/com/linkedin/openhouse/spark/builder/PartitionSpecBuilder.java
diff --git a/settings.gradle b/settings.gradle
@@ -61,6 +61,10 @@ include ':cluster:metrics'
 include ':scripts:java:tools:dummytokens'
 
 project(':integrations:spark:openhouse-spark-runtime').name = 'openhouse-spark-runtime_2.12'
-project(':integrations:spark-3.5:openhouse-spark-runtime').name = 'openhouse-spark-3.5-runtime_2.12'
 project(':apps:spark').name = 'openhouse-spark-apps_2.12'
 project(':tables-test-fixtures').name = 'tables-test-fixtures_2.12'
+
+project(':integrations:spark-3.5:openhouse-spark-runtime').name = 'openhouse-spark-3.5-runtime_2.12'
+project(':integrations:spark-3.5:openhouse-spark-itest').name = 'openhouse-spark-3.5-itest'
+project(':integrations:java-iceberg-1.5:openhouse-java-runtime').name = 'openhouse-java-iceberg-1.5-runtime'
+project(':integrations:java-iceberg-1.5:openhouse-java-itest').name = 'openhouse-java-iceberg-1.5-itest'