From f8c386e512fc8ebf75a337833ac38ba87629f903 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Tue, 15 Oct 2024 16:06:14 -0500 Subject: [PATCH] Change DataSource calendar interval error to fix spark400 build (#11610) Signed-off-by: Jason Lowe --- .../spark/sql/rapids/shims/GpuDataSource.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuDataSource.scala b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuDataSource.scala index 37395a72d2f..c05cc0dbc60 100644 --- a/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuDataSource.scala +++ b/sql-plugin/src/main/spark332db/scala/org/apache/spark/sql/rapids/shims/GpuDataSource.scala @@ -121,16 +121,18 @@ case class GpuDataSource( data: LogicalPlan, outputColumnNames: Seq[String]): BaseRelation = { - val outputColumns = DataWritingCommand.logicalPlanOutputWithNames(data, outputColumnNames) - if (outputColumns.map(_.dataType).exists(_.isInstanceOf[CalendarIntervalType])) { - throw QueryCompilationErrors.cannotSaveIntervalIntoExternalStorageError() - } - val format = originalProvidingInstance() if (!format.isInstanceOf[FileFormat]) { throw new IllegalArgumentException(s"Original provider does not extend FileFormat: $format") } + val outputColumns = DataWritingCommand.logicalPlanOutputWithNames(data, outputColumnNames) + outputColumns.toStructType.foreach { field => + if (field.dataType.isInstanceOf[CalendarIntervalType]) { + throw QueryCompilationErrors.dataTypeUnsupportedByDataSourceError(format.toString, field) + } + } + val cmd = planForWritingFileFormat(format.asInstanceOf[FileFormat], mode, data) // Spark 3.4 doesn't need the child physical plan for metrics anymore, this is now // cleaned up, so we need to run the DataWritingCommand using SparkSession. This actually