revert experimental code

NVIDIA · Oct 5, 2023 · 34b737d · 34b737d
1 parent a658d05
commit 34b737d
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 109 deletions.
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCSVScan.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCSVScan.scala
@@ -195,6 +195,20 @@ object GpuCSVScan {
         s"To enable it please set ${RapidsConf.ENABLE_READ_CSV_DECIMALS} to true.")
     }
 
+    // TODO move this check to 350 shim
+    if (parsedOptions.inferSchemaFlag) {
+      parsedOptions.timestampFormatInRead match {
+        case Some(fmt) => fmt match {
+          case "yyyy-MM-dd" | "yyyy-MM" | "yyyy-MM-dd'T'HH:mm" | "yyyy-MM-dd'T'HH:mm:ss" =>
+            // https://github.com/NVIDIA/spark-rapids/issues/9325
+            meta.willNotWorkOnGpu(s"timestampFormat '$fmt' is not compatible with " +
+              s"Spark >= 3.5.0 when schema inference is enabled")
+          case _ =>
+        }
+        case _ =>
+      }
+    }
+
     FileFormatChecks.tag(meta, readSchema, CsvFormatType, ReadFileOp)
   }
 }

diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/BatchScanExecMeta.scala
@@ -24,7 +24,6 @@ import com.nvidia.spark.rapids._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
-import org.apache.spark.sql.execution.datasources.v2.csv.CSVScan
 
 class BatchScanExecMeta(p: BatchScanExec,
     conf: RapidsConf,
@@ -70,15 +69,6 @@ class BatchScanExecMeta(p: BatchScanExec,
     if (!p.runtimeFilters.isEmpty && !childScans.head.supportsRuntimeFilters) {
       willNotWorkOnGpu("runtime filtering (DPP) is not supported for this scan")
     }
-
-    println(s"ANDY BatchScanExecMeta.tagSelfForGpu()")
-
-    p.scan match {
-      case _: CSVScan =>
-        println("ANDY CSV")
-      case _ =>
-        println("ANDY NOT CSV")
-    }
   }
 
   override def convertToCpu(): SparkPlan = {

diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/RapidsCsvScanMeta.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/RapidsCsvScanMeta.scala
diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/Spark350PlusShims.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/Spark350PlusShims.scala
diff --git a/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/SparkShims.scala b/sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/SparkShims.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, PythonUDAF, ToPret
 import org.apache.spark.sql.rapids.execution.python.GpuPythonUDAF
 import org.apache.spark.sql.types.StringType
 
-object SparkShimImpl extends Spark350PlusShims {
+object SparkShimImpl extends Spark340PlusShims {
 
   override def getExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = {
     val shimExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Seq(