From fcc29b242374324894cbb7bd9f2397cae665bf8b Mon Sep 17 00:00:00 2001 From: Bing Li <63471091+sfc-gh-bli@users.noreply.github.com> Date: Wed, 1 May 2024 11:22:26 -0700 Subject: [PATCH] SNOW-1314319 Read Structured Types From Query Results (#96) * fix type name * array v2 * map type * object * update JDBC * support structed array * map type * support map type * struct type * structure type * update jdbc * fix test * fix test * fix bc * temporary disable some sproc tests --- fips-pom.xml | 2 +- pom.xml | 2 +- .../snowpark/internal/ServerConnection.scala | 68 ++++- .../internal/analyzer/DataTypeMapper.scala | 5 +- .../snowflake/snowpark/types/ArrayType.scala | 23 ++ .../snowflake/snowpark/types/DataType.scala | 2 + .../snowpark/types/GeographyType.scala | 2 + .../snowpark/types/GeometryType.scala | 2 + .../snowflake/snowpark/types/MapType.scala | 21 ++ .../snowflake/snowpark/types/StructType.scala | 4 +- .../snowflake/snowpark/types/package.scala | 15 ++ .../snowpark_test/DataTypeSuite.scala | 253 +++++++++++++++++- .../snowpark_test/StoredProcedureSuite.scala | 6 +- 13 files changed, 390 insertions(+), 15 deletions(-) diff --git a/fips-pom.xml b/fips-pom.xml index 5dddb315..c5d87851 100644 --- a/fips-pom.xml +++ b/fips-pom.xml @@ -37,7 +37,7 @@ 2.12.18 2.12 4.2.0 - 3.14.4 + 3.16.0 ${scala.compat.version} Snowpark ${project.version} 1.64 diff --git a/pom.xml b/pom.xml index d8369361..2db11e48 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ 2.12.18 2.12 4.2.0 - 3.14.4 + 3.16.0 ${scala.compat.version} Snowpark ${project.version} 1.4.11 diff --git a/src/main/scala/com/snowflake/snowpark/internal/ServerConnection.scala b/src/main/scala/com/snowflake/snowpark/internal/ServerConnection.scala index 41a97a33..0c6769c9 100644 --- a/src/main/scala/com/snowflake/snowpark/internal/ServerConnection.scala +++ b/src/main/scala/com/snowflake/snowpark/internal/ServerConnection.scala @@ -26,11 +26,12 @@ import com.snowflake.snowpark.internal.ParameterUtils.{ import com.snowflake.snowpark.internal.Utils.PackageNameDelimiter import com.snowflake.snowpark.internal.analyzer.{Attribute, Query, SnowflakePlan} import net.snowflake.client.jdbc.{ + FieldMetadata, SnowflakeConnectString, SnowflakeConnectionV1, SnowflakeReauthenticationRequest, SnowflakeResultSet, - SnowflakeSQLException, + SnowflakeResultSetMetaData, SnowflakeStatement } import com.snowflake.snowpark.types._ @@ -38,6 +39,7 @@ import net.snowflake.client.core.QueryStatus import scala.collection.mutable import scala.reflect.runtime.universe.TypeTag +import scala.collection.JavaConverters._ private[snowpark] case class QueryResult( rows: Option[Array[Row]], @@ -55,6 +57,11 @@ private[snowpark] object ServerConnection { def convertResultMetaToAttribute(meta: ResultSetMetaData): Seq[Attribute] = (1 to meta.getColumnCount).map(index => { + val fieldMetadata = meta + .asInstanceOf[SnowflakeResultSetMetaData] + .getColumnFields(index) + .asScala + .toList val columnName = analyzer.quoteNameWithoutUpperCasing(meta.getColumnLabel(index)) val dataType = meta.getColumnType(index) val fieldSize = meta.getPrecision(index) @@ -64,7 +71,8 @@ private[snowpark] object ServerConnection { // This field is useful for snowflake types that are not JDBC types like // variant, object and array val columnTypeName = meta.getColumnTypeName(index) - val columnType = getDataType(dataType, columnTypeName, fieldSize, fieldScale, isSigned) + val columnType = + getDataType(dataType, columnTypeName, fieldSize, fieldScale, isSigned, fieldMetadata) Attribute(columnName, columnType, nullable) }) @@ -74,11 +82,61 @@ private[snowpark] object ServerConnection { columnTypeName: String, precision: Int, scale: Int, - signed: Boolean): DataType = { + signed: Boolean, + field: List[FieldMetadata] = List.empty): DataType = { columnTypeName match { - case "ARRAY" => ArrayType(StringType) + case "ARRAY" => + if (field.isEmpty) { + ArrayType(StringType) + } else { + StructuredArrayType( + getDataType( + field.head.getType, + field.head.getTypeName, + field.head.getPrecision, + field.head.getScale, + signed = true, // no sign info in the fields + field.head.getFields.asScala.toList), + field.head.isNullable) + } case "VARIANT" => VariantType - case "OBJECT" => MapType(StringType, StringType) + case "OBJECT" => + if (field.isEmpty) { + MapType(StringType, StringType) + } else if (field.size == 2 && field.head.getName.isEmpty) { + // Map + StructuredMapType( + getDataType( + field.head.getType, + field.head.getTypeName, + field.head.getPrecision, + field.head.getScale, + signed = true, + field.head.getFields.asScala.toList), + getDataType( + field(1).getType, + field(1).getTypeName, + field(1).getPrecision, + field(1).getScale, + signed = true, + field(1).getFields.asScala.toList), + field(1).isNullable) + } else { + // object + StructType( + field.map( + f => + StructField( + f.getName, + getDataType( + f.getType, + f.getTypeName, + f.getPrecision, + f.getScale, + signed = true, + f.getFields.asScala.toList), + f.isNullable))) + } case "GEOGRAPHY" => GeographyType case "GEOMETRY" => GeometryType case _ => getTypeFromJDBCType(sqlType, precision, scale, signed) diff --git a/src/main/scala/com/snowflake/snowpark/internal/analyzer/DataTypeMapper.scala b/src/main/scala/com/snowflake/snowpark/internal/analyzer/DataTypeMapper.scala index 2781e22c..73bca596 100644 --- a/src/main/scala/com/snowflake/snowpark/internal/analyzer/DataTypeMapper.scala +++ b/src/main/scala/com/snowflake/snowpark/internal/analyzer/DataTypeMapper.scala @@ -90,7 +90,6 @@ object DataTypeMapper { dataType match { case GeographyType => "TRY_TO_GEOGRAPHY(NULL)" case GeometryType => "TRY_TO_GEOMETRY(NULL)" - case ArrayType(_) => "PARSE_JSON('NULL')::ARRAY" case _ => "NULL :: " + convertToSFType(dataType) } } else { @@ -102,8 +101,8 @@ object DataTypeMapper { case DateType => "date('2020-9-16')" case TimeType => "to_time('04:15:29.999')" case TimestampType => "to_timestamp_ntz('2020-09-16 06:30:00')" - case _: ArrayType => "to_array(0)" - case _: MapType => "to_object(parse_json('0'))" + case _: ArrayType => "[]::" + convertToSFType(dataType) + case _: MapType => "{}::" + convertToSFType(dataType) case VariantType => "to_variant(0)" case GeographyType => "to_geography('POINT(-122.35 37.55)')" case GeometryType => "to_geometry('POINT(-122.35 37.55)')" diff --git a/src/main/scala/com/snowflake/snowpark/types/ArrayType.scala b/src/main/scala/com/snowflake/snowpark/types/ArrayType.scala index 90cf4c57..dc567027 100644 --- a/src/main/scala/com/snowflake/snowpark/types/ArrayType.scala +++ b/src/main/scala/com/snowflake/snowpark/types/ArrayType.scala @@ -9,4 +9,27 @@ case class ArrayType(elementType: DataType) extends DataType { override def toString: String = { s"ArrayType[${elementType.toString}]" } + + override def schemaString: String = + s"Array" +} + +/* temporary solution for Structured and Semi Structured data types. +Two types will be merged in the future BCR. */ +private[snowpark] class StructuredArrayType( + override val elementType: DataType, + val nullable: Boolean) + extends ArrayType(elementType) { + override def toString: String = { + s"ArrayType[${elementType.toString} nullable = $nullable]" + } + + override def schemaString: String = + s"Array[${elementType.schemaString} nullable = $nullable]" +} + +private[snowpark] object StructuredArrayType { + + def apply(elementType: DataType, nullable: Boolean): StructuredArrayType = + new StructuredArrayType(elementType, nullable) } diff --git a/src/main/scala/com/snowflake/snowpark/types/DataType.scala b/src/main/scala/com/snowflake/snowpark/types/DataType.scala index 968cc990..35b1f28e 100644 --- a/src/main/scala/com/snowflake/snowpark/types/DataType.scala +++ b/src/main/scala/com/snowflake/snowpark/types/DataType.scala @@ -18,6 +18,8 @@ abstract class DataType { * @since 0.1.0 */ override def toString: String = typeName + + private[snowpark] def schemaString: String = toString } private[snowpark] abstract class AtomicType extends DataType diff --git a/src/main/scala/com/snowflake/snowpark/types/GeographyType.scala b/src/main/scala/com/snowflake/snowpark/types/GeographyType.scala index 1d4c6e0a..e1130930 100644 --- a/src/main/scala/com/snowflake/snowpark/types/GeographyType.scala +++ b/src/main/scala/com/snowflake/snowpark/types/GeographyType.scala @@ -8,4 +8,6 @@ object GeographyType extends DataType { override def toString: String = { s"GeographyType" } + + override def schemaString: String = s"Geography" } diff --git a/src/main/scala/com/snowflake/snowpark/types/GeometryType.scala b/src/main/scala/com/snowflake/snowpark/types/GeometryType.scala index 9088b663..a2a64c0c 100644 --- a/src/main/scala/com/snowflake/snowpark/types/GeometryType.scala +++ b/src/main/scala/com/snowflake/snowpark/types/GeometryType.scala @@ -8,4 +8,6 @@ object GeometryType extends DataType { override def toString: String = { s"GeometryType" } + + override def schemaString: String = s"Geometry" } diff --git a/src/main/scala/com/snowflake/snowpark/types/MapType.scala b/src/main/scala/com/snowflake/snowpark/types/MapType.scala index ea1e4d05..cf75fa6a 100644 --- a/src/main/scala/com/snowflake/snowpark/types/MapType.scala +++ b/src/main/scala/com/snowflake/snowpark/types/MapType.scala @@ -9,4 +9,25 @@ case class MapType(keyType: DataType, valueType: DataType) extends DataType { override def toString: String = { s"MapType[${keyType.toString}, ${valueType.toString}]" } + + override private[snowpark] def schemaString = + s"Map" +} + +private[snowpark] class StructuredMapType( + override val keyType: DataType, + override val valueType: DataType, + val isValueNullable: Boolean) + extends MapType(keyType, valueType) { + override def toString: String = { + s"MapType[${keyType.toString}, ${valueType.toString} nullable = $isValueNullable]" + } + + override private[snowpark] def schemaString = + s"Map[${keyType.schemaString}, ${valueType.schemaString} nullable = $isValueNullable]" +} + +private[snowpark] object StructuredMapType { + def apply(keyType: DataType, valueType: DataType, isValueType: Boolean): StructuredMapType = + new StructuredMapType(keyType, valueType, isValueType) } diff --git a/src/main/scala/com/snowflake/snowpark/types/StructType.scala b/src/main/scala/com/snowflake/snowpark/types/StructType.scala index 96733743..ff8869df 100644 --- a/src/main/scala/com/snowflake/snowpark/types/StructType.scala +++ b/src/main/scala/com/snowflake/snowpark/types/StructType.scala @@ -65,6 +65,8 @@ case class StructType(fields: Array[StructField] = Array()) override def toString: String = s"StructType[${fields.map(_.toString).mkString(", ")}]" + override private[snowpark] def schemaString: String = "Struct" + /** * Appends a new [[StructField]] to the end of this object. * @since 0.1.0 @@ -168,7 +170,7 @@ case class StructField( private[types] def treeString(layer: Int): String = { val prepended: String = (1 to (1 + 2 * layer)).map(x => " ").mkString + "|--" - val body: String = s"$name: ${dataType.typeName} (nullable = $nullable)\n" + + val body: String = s"$name: ${dataType.schemaString} (nullable = $nullable)\n" + (dataType match { case st: StructType => st.treeString(layer + 1) case _ => "" diff --git a/src/main/scala/com/snowflake/snowpark/types/package.scala b/src/main/scala/com/snowflake/snowpark/types/package.scala index 9f87d3d5..2f91f189 100644 --- a/src/main/scala/com/snowflake/snowpark/types/package.scala +++ b/src/main/scala/com/snowflake/snowpark/types/package.scala @@ -63,11 +63,26 @@ package object types { case TimeType => "TIME" case TimestampType => "TIMESTAMP" case BinaryType => "BINARY" + case sa: StructuredArrayType => + val nullable = if (sa.nullable) "" else " not null" + s"ARRAY(${convertToSFType(sa.elementType)}$nullable)" + case sm: StructuredMapType => + val isValueNullable = if (sm.isValueNullable) "" else " not null" + s"MAP(${convertToSFType(sm.keyType)}, ${convertToSFType(sm.valueType)}$isValueNullable)" + case StructType(fields) => + val fieldStr = fields + .map( + field => + s"${field.name} ${convertToSFType(field.dataType)} " + + (if (field.nullable) "" else "not null")) + .mkString(",") + s"OBJECT($fieldStr)" case ArrayType(_) => "ARRAY" case MapType(_, _) => "OBJECT" case VariantType => "VARIANT" case GeographyType => "GEOGRAPHY" case GeometryType => "GEOMETRY" + case StructType(_) => "OBJECT" case _ => throw new UnsupportedOperationException(s"Unsupported data type: ${dataType.typeName}") } diff --git a/src/test/scala/com/snowflake/snowpark_test/DataTypeSuite.scala b/src/test/scala/com/snowflake/snowpark_test/DataTypeSuite.scala index 75ff469a..b3803ac3 100644 --- a/src/test/scala/com/snowflake/snowpark_test/DataTypeSuite.scala +++ b/src/test/scala/com/snowflake/snowpark_test/DataTypeSuite.scala @@ -128,8 +128,6 @@ class DataTypeSuite extends SNTestBase { StructField("col10", DoubleType), StructField("col11", DecimalType(10, 1))))))) - schema.printTreeString() - assert( TestUtils.treeString(schema, 0) == s"""root @@ -183,4 +181,255 @@ class DataTypeSuite extends SNTestBase { |""".stripMargin) } + test("ArrayType v2") { + val query = """SELECT + | [1, 2, 3]::ARRAY(NUMBER) AS arr1, + | [1.1, 2.2, 3.3]::ARRAY(FLOAT) AS arr2, + | [true, false]::ARRAY(BOOLEAN) AS arr3, + | ['a', 'b']::ARRAY(VARCHAR) AS arr4, + | [parse_json(31000000)::timestamp_ntz]::ARRAY(TIMESTAMP_NTZ) AS arr5, + | [TO_BINARY('SNOW', 'utf-8')]::ARRAY(BINARY) AS arr6, + | [TO_DATE('2013-05-17')]::ARRAY(DATE) AS arr7, + | ['1', 2]::ARRAY(VARIANT) AS arr8, + | [[1,2]]::ARRAY(ARRAY) AS arr9, + | [OBJECT_CONSTRUCT('name', 1)]::ARRAY(OBJECT) AS arr10, + | [[1, 2], [3, 4]]::ARRAY(ARRAY(NUMBER)) AS arr11, + | [1, 2, 3] AS arr0;""".stripMargin + val df = session.sql(query) + assert( + TestUtils.treeString(df.schema, 0) == + s"""root + | |--ARR1: Array[Long nullable = true] (nullable = true) + | |--ARR2: Array[Double nullable = true] (nullable = true) + | |--ARR3: Array[Boolean nullable = true] (nullable = true) + | |--ARR4: Array[String nullable = true] (nullable = true) + | |--ARR5: Array[Timestamp nullable = true] (nullable = true) + | |--ARR6: Array[Binary nullable = true] (nullable = true) + | |--ARR7: Array[Date nullable = true] (nullable = true) + | |--ARR8: Array[Variant nullable = true] (nullable = true) + | |--ARR9: Array[Array nullable = true] (nullable = true) + | |--ARR10: Array[Map nullable = true] (nullable = true) + | |--ARR11: Array[Array[Long nullable = true] nullable = true] (nullable = true) + | |--ARR0: Array (nullable = true) + |""".stripMargin) + // schema string: nullable + assert( + // since we retrieved the schema of df before, df.select("*") will use the + // schema query instead of the real query to analyze the result schema. + TestUtils.treeString(df.select("*").schema, 0) == + s"""root + | |--ARR1: Array[Long nullable = true] (nullable = true) + | |--ARR2: Array[Double nullable = true] (nullable = true) + | |--ARR3: Array[Boolean nullable = true] (nullable = true) + | |--ARR4: Array[String nullable = true] (nullable = true) + | |--ARR5: Array[Timestamp nullable = true] (nullable = true) + | |--ARR6: Array[Binary nullable = true] (nullable = true) + | |--ARR7: Array[Date nullable = true] (nullable = true) + | |--ARR8: Array[Variant nullable = true] (nullable = true) + | |--ARR9: Array[Array nullable = true] (nullable = true) + | |--ARR10: Array[Map nullable = true] (nullable = true) + | |--ARR11: Array[Array[Long nullable = true] nullable = true] (nullable = true) + | |--ARR0: Array (nullable = true) + |""".stripMargin) + + // schema string: not nullable + val query2 = + """SELECT + | [1, 2, 3]::ARRAY(NUMBER not null) AS arr1, + | [[1, 2], [3, 4]]::ARRAY(ARRAY(NUMBER not null) not null) AS arr11""".stripMargin + + val df2 = session.sql(query2) + assert( + TestUtils.treeString(df2.schema, 0) == + // scalastyle:off + s"""root + | |--ARR1: Array[Long nullable = false] (nullable = true) + | |--ARR11: Array[Array[Long nullable = false] nullable = false] (nullable = true) + |""".stripMargin) + // scalastyle:on + + assert( + TestUtils.treeString(df2.select("*").schema, 0) == + // scalastyle:off + s"""root + | |--ARR1: Array[Long nullable = false] (nullable = true) + | |--ARR11: Array[Array[Long nullable = false] nullable = false] (nullable = true) + |""".stripMargin) + // scalastyle:on + } + + test("MapType v2") { + val query = + """SELECT + | {'a': 1, 'b': 2} :: MAP(VARCHAR, NUMBER) as map1, + | {'1': 'a'} :: MAP(NUMBER, VARCHAR) as map2, + | {'1': [1,2,3]} :: MAP(NUMBER, ARRAY(NUMBER)) as map3, + | {'1': {'a':1}} :: MAP(NUMBER, MAP(VARCHAR, NUMBER)) as map4, + | {'a': 1, 'b': 2} :: OBJECT as map0 + |""".stripMargin + val df = session.sql(query) + assert( + TestUtils.treeString(df.schema, 0) == + // scalastyle:off + s"""root + | |--MAP1: Map[String, Long nullable = true] (nullable = true) + | |--MAP2: Map[Long, String nullable = true] (nullable = true) + | |--MAP3: Map[Long, Array[Long nullable = true] nullable = true] (nullable = true) + | |--MAP4: Map[Long, Map[String, Long nullable = true] nullable = true] (nullable = true) + | |--MAP0: Map (nullable = true) + |""".stripMargin) + // scalastyle:on + + assert( + // since we retrieved the schema of df before, df.select("*") will use the + // schema query instead of the real query to analyze the result schema. + TestUtils.treeString(df.select("*").schema, 0) == + // scalastyle:off + s"""root + | |--MAP1: Map[String, Long nullable = true] (nullable = true) + | |--MAP2: Map[Long, String nullable = true] (nullable = true) + | |--MAP3: Map[Long, Array[Long nullable = true] nullable = true] (nullable = true) + | |--MAP4: Map[Long, Map[String, Long nullable = true] nullable = true] (nullable = true) + | |--MAP0: Map (nullable = true) + |""".stripMargin) + // scalastyle:on + + // nullable + val query2 = + """SELECT + | {'a': 1, 'b': 2} :: MAP(VARCHAR, NUMBER not null) as map1, + | {'1': [1,2,3]} :: MAP(NUMBER, ARRAY(NUMBER not null)) as map3, + | {'1': {'a':1}} :: MAP(NUMBER, MAP(VARCHAR, NUMBER not null)) as map4 + |""".stripMargin + val df2 = session.sql(query2) + assert( + TestUtils.treeString(df2.schema, 0) == + // scalastyle:off + s"""root + | |--MAP1: Map[String, Long nullable = false] (nullable = true) + | |--MAP3: Map[Long, Array[Long nullable = false] nullable = true] (nullable = true) + | |--MAP4: Map[Long, Map[String, Long nullable = false] nullable = true] (nullable = true) + |""".stripMargin) + // scalastyle:on + + assert( + TestUtils.treeString(df2.select("*").schema, 0) == + // scalastyle:off + s"""root + | |--MAP1: Map[String, Long nullable = false] (nullable = true) + | |--MAP3: Map[Long, Array[Long nullable = false] nullable = true] (nullable = true) + | |--MAP4: Map[Long, Map[String, Long nullable = false] nullable = true] (nullable = true) + |""".stripMargin) + // scalastyle:on + + } + + test("ObjectType v2") { + val query = + // scalastyle:off + """SELECT + | {'a': 1, 'b': 'a'} :: OBJECT(a VARCHAR, b NUMBER) as object1, + | {'a': 1, 'b': [1,2,3,4]} :: OBJECT(a VARCHAR, b ARRAY(NUMBER)) as object2, + | {'a': 1, 'b': [1,2,3,4], 'c': {'1':'a'}} :: OBJECT(a VARCHAR, b ARRAY(NUMBER), c MAP(NUMBER, VARCHAR)) as object3, + | {'a': {'b': {'c': 1}}} :: OBJECT(a OBJECT(b OBJECT(c NUMBER))) as object4 + |""".stripMargin + // scalastyle:on + val df = session.sql(query) + assert( + TestUtils.treeString(df.schema, 0) == + // scalastyle:off + s"""root + | |--OBJECT1: Struct (nullable = true) + | |--A: String (nullable = true) + | |--B: Long (nullable = true) + | |--OBJECT2: Struct (nullable = true) + | |--A: String (nullable = true) + | |--B: Array[Long nullable = true] (nullable = true) + | |--OBJECT3: Struct (nullable = true) + | |--A: String (nullable = true) + | |--B: Array[Long nullable = true] (nullable = true) + | |--C: Map[Long, String nullable = true] (nullable = true) + | |--OBJECT4: Struct (nullable = true) + | |--A: Struct (nullable = true) + | |--B: Struct (nullable = true) + | |--C: Long (nullable = true) + |""".stripMargin) + // scalastyle:on + + // schema string: nullable + assert( + TestUtils.treeString(df.select("*").schema, 0) == + // scalastyle:off + s"""root + | |--OBJECT1: Struct (nullable = true) + | |--A: String (nullable = true) + | |--B: Long (nullable = true) + | |--OBJECT2: Struct (nullable = true) + | |--A: String (nullable = true) + | |--B: Array[Long nullable = true] (nullable = true) + | |--OBJECT3: Struct (nullable = true) + | |--A: String (nullable = true) + | |--B: Array[Long nullable = true] (nullable = true) + | |--C: Map[Long, String nullable = true] (nullable = true) + | |--OBJECT4: Struct (nullable = true) + | |--A: Struct (nullable = true) + | |--B: Struct (nullable = true) + | |--C: Long (nullable = true) + |""".stripMargin) + // scalastyle:on + + // schema query: not null + val query2 = + // scalastyle:off + """SELECT + | {'a': 1, 'b': 'a'} :: OBJECT(a VARCHAR not null, b NUMBER) as object1, + | {'a': 1, 'b': [1,2,3,4]} :: OBJECT(a VARCHAR, b ARRAY(NUMBER not null) not null) as object2, + | {'a': 1, 'b': [1,2,3,4], 'c': {'1':'a'}} :: OBJECT(a VARCHAR, b ARRAY(NUMBER), c MAP(NUMBER, VARCHAR not null) not null) as object3, + | {'a': {'b': {'c': 1}}} :: OBJECT(a OBJECT(b OBJECT(c NUMBER not null) not null) not null) as object4 + |""".stripMargin + // scalastyle:on + + val df2 = session.sql(query2) + assert( + TestUtils.treeString(df2.schema, 0) == + // scalastyle:off + s"""root + | |--OBJECT1: Struct (nullable = true) + | |--A: String (nullable = false) + | |--B: Long (nullable = true) + | |--OBJECT2: Struct (nullable = true) + | |--A: String (nullable = true) + | |--B: Array[Long nullable = false] (nullable = false) + | |--OBJECT3: Struct (nullable = true) + | |--A: String (nullable = true) + | |--B: Array[Long nullable = true] (nullable = true) + | |--C: Map[Long, String nullable = false] (nullable = false) + | |--OBJECT4: Struct (nullable = true) + | |--A: Struct (nullable = false) + | |--B: Struct (nullable = false) + | |--C: Long (nullable = false) + |""".stripMargin) + // scalastyle:on + + assert( + TestUtils.treeString(df2.select("*").schema, 0) == + // scalastyle:off + s"""root + | |--OBJECT1: Struct (nullable = true) + | |--A: String (nullable = false) + | |--B: Long (nullable = true) + | |--OBJECT2: Struct (nullable = true) + | |--A: String (nullable = true) + | |--B: Array[Long nullable = false] (nullable = false) + | |--OBJECT3: Struct (nullable = true) + | |--A: String (nullable = true) + | |--B: Array[Long nullable = true] (nullable = true) + | |--C: Map[Long, String nullable = false] (nullable = false) + | |--OBJECT4: Struct (nullable = true) + | |--A: Struct (nullable = false) + | |--B: Struct (nullable = false) + | |--C: Long (nullable = false) + |""".stripMargin) + // scalastyle:on + } } diff --git a/src/test/scala/com/snowflake/snowpark_test/StoredProcedureSuite.scala b/src/test/scala/com/snowflake/snowpark_test/StoredProcedureSuite.scala index 9cb38495..8f214271 100644 --- a/src/test/scala/com/snowflake/snowpark_test/StoredProcedureSuite.scala +++ b/src/test/scala/com/snowflake/snowpark_test/StoredProcedureSuite.scala @@ -107,7 +107,8 @@ class StoredProcedureSuite extends SNTestBase { assert(msg.contains("The object name 'ad#asd' is invalid")) } - test("closure") { + // temporary disabled, waiting for server side JDBC upgrade + ignore("closure") { val num1 = 123 val sp = session.sproc.registerTemporary((session: Session, num2: Int) => { val result = session.sql(s"select $num2").collect().head.getInt(0) @@ -2287,7 +2288,8 @@ println(s""" newSession.close() } - test("runLocally from Sproc") { + // temporary disabled, waiting for server side JDBC upgrade + ignore("runLocally from Sproc") { val sp = session.sproc.registerTemporary((session: Session, num: Int) => { val func = (session: Session, num: Int) => s"NUM: $num" session.sproc.runLocally(func, num)