From ad4233d1ceb0cc79ad9567b5079ec9bdaccc83ff Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Wed, 6 Nov 2024 08:42:53 -0600 Subject: [PATCH] Fix spark400 build due to LogicalRelation signature changes (#11695) Signed-off-by: Jason Lowe --- .../spark/rapids/shims/LogicalPlanShims.scala | 38 +++++++++++++++++++ .../sql/rapids/shims/GpuDataSource.scala | 5 +-- .../spark/rapids/shims/LogicalPlanShims.scala | 29 ++++++++++++++ 3 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/LogicalPlanShims.scala create mode 100644 sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/LogicalPlanShims.scala diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/LogicalPlanShims.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/LogicalPlanShims.scala new file mode 100644 index 00000000000..d0f3d0f7532 --- /dev/null +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/LogicalPlanShims.scala @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "332db"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +{"spark": "352"} +{"spark": "353"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.execution.datasources.{FileIndex, HadoopFsRelation, LogicalRelation} + +object LogicalPlanShims { + def getLocations(plan: LogicalPlan): Seq[FileIndex] = plan.collect { + case LogicalRelation(t: HadoopFsRelation, _, _, _) => t.location + } +} diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuDataSource.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuDataSource.scala index 58aa6739bad..05534a39a8c 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuDataSource.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuDataSource.scala @@ -29,6 +29,7 @@ spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids +import com.nvidia.spark.rapids.shims.LogicalPlanShims import org.apache.hadoop.fs.Path import org.apache.spark.sql._ @@ -83,9 +84,7 @@ case class GpuDataSource( PartitioningUtils.validatePartitionColumn(data.schema, partitionColumns, caseSensitive) val fileIndex = catalogTable.map(_.identifier).map { tableIdent => - sparkSession.table(tableIdent).queryExecution.analyzed.collect { - case LogicalRelation(t: HadoopFsRelation, _, _, _) => t.location - }.head + LogicalPlanShims.getLocations(sparkSession.table(tableIdent).queryExecution.analyzed).head } // For partitioned relation r, r.schema's column ordering can be different from the column diff --git a/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/LogicalPlanShims.scala b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/LogicalPlanShims.scala new file mode 100644 index 00000000000..4379c5e1f1d --- /dev/null +++ b/sql-plugin/src/main/spark400/scala/com/nvidia/spark/rapids/shims/LogicalPlanShims.scala @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package com.nvidia.spark.rapids.shims + +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.execution.datasources.{FileIndex, HadoopFsRelation, LogicalRelationWithTable} + +object LogicalPlanShims { + def getLocations(plan: LogicalPlan): Seq[FileIndex] = plan.collect { + case LogicalRelationWithTable(t: HadoopFsRelation, _) => t.location + } +}