From a44006c99832ed59c714604bdabe33711cf4c4b1 Mon Sep 17 00:00:00 2001 From: Abhishek Radhakrishnan Date: Fri, 18 Oct 2024 08:09:35 -0700 Subject: [PATCH] Fix decimal type support for the delta input format. (#17376) The Delta Decimal type wasn't handled correctly in the Druid Delta connector, resulting in the error: Unsupported fieldType[Decimal(4, 2)] for fieldName[price]. There were no tests or existing tables with the Decimal type, so I've updated the existing table, complex-types-table to include this data type. Note that the Decimal type can only be handled as a double at most in Druid. For a big decimal that cannot fit inside a double, it should be ingested as a string. --- .../apache/druid/delta/input/DeltaInputRow.java | 2 +- .../org/apache/druid/delta/input/RowSerde.java | 3 +++ .../delta/input/ComplexTypesDeltaTable.java | 11 ++++++----- ...d3e-8bff-04182c482c2f-c000.snappy.parquet.crc | Bin 0 -> 24 bytes ...a53-9b74-0cc7b853103a-c000.snappy.parquet.crc | Bin 20 -> 0 bytes ...e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc | Bin 36 -> 0 bytes ...b19-bce2-303feb8f9c66-c000.snappy.parquet.crc | Bin 0 -> 40 bytes ...b08-abf3-e4f79a9062f5-c000.snappy.parquet.crc | Bin 0 -> 40 bytes ...ef3-82a4-f3f273308e53-c000.snappy.parquet.crc | Bin 36 -> 0 bytes ...27d-ad22-7036bb052433-c000.snappy.parquet.crc | Bin 0 -> 40 bytes ...04a-bb38-f627c47eb20b-c000.snappy.parquet.crc | Bin 36 -> 0 bytes ...141-bc77-0106e7f28f7a-c000.snappy.parquet.crc | Bin 36 -> 0 bytes ...b92-a61f-b9a3e5d8b986-c000.snappy.parquet.crc | Bin 0 -> 40 bytes ...3fe-b605-506c942cd969-c000.snappy.parquet.crc | Bin 36 -> 0 bytes ...526-abc6-69042b415d87-c000.snappy.parquet.crc | Bin 0 -> 40 bytes .../_delta_log/.00000000000000000000.json.crc | Bin 52 -> 56 bytes .../_delta_log/00000000000000000000.json | 14 +++++++------- ...e4-4d3e-8bff-04182c482c2f-c000.snappy.parquet | Bin 0 -> 1604 bytes ...85-4a53-9b74-0cc7b853103a-c000.snappy.parquet | Bin 1488 -> 0 bytes ...71-4e91-834e-2a1cb6601eb8-c000.snappy.parquet | Bin 3288 -> 0 bytes ...79-4b19-bce2-303feb8f9c66-c000.snappy.parquet | Bin 0 -> 3588 bytes ...0e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet | Bin 0 -> 3591 bytes ...24-4ef3-82a4-f3f273308e53-c000.snappy.parquet | Bin 3291 -> 0 bytes ...0a-427d-ad22-7036bb052433-c000.snappy.parquet | Bin 0 -> 3589 bytes ...89-404a-bb38-f627c47eb20b-c000.snappy.parquet | Bin 3289 -> 0 bytes ...f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet | Bin 3290 -> 0 bytes ...26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet | Bin 0 -> 3590 bytes ...ce-43fe-b605-506c942cd969-c000.snappy.parquet | Bin 3291 -> 0 bytes ...c7-4526-abc6-69042b415d87-c000.snappy.parquet | Bin 0 -> 3591 bytes .../src/test/resources/create_delta_table.py | 7 ++++--- 30 files changed, 21 insertions(+), 16 deletions(-) create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet.crc delete mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc delete mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet.crc create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet.crc delete mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet.crc delete mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc delete mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet.crc delete mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet.crc create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet delete mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet delete mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet delete mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet delete mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet delete mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet delete mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet diff --git a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java index 442412dd154f..a8cd0d5020e2 100644 --- a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java +++ b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java @@ -201,7 +201,7 @@ private static Object getValue(DataType dataType, io.delta.kernel.data.Row dataR } return String.valueOf(charArray); } else if (dataType instanceof DecimalType) { - return dataRow.getDecimal(columnOrdinal).longValue(); + return dataRow.getDecimal(columnOrdinal); } else if (dataType instanceof StructType) { final io.delta.kernel.data.Row structRow = dataRow.getStruct(columnOrdinal); return RowSerde.convertRowToJsonObject(structRow); diff --git a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/RowSerde.java b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/RowSerde.java index bad6191496d7..d7c6fcccdba2 100644 --- a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/RowSerde.java +++ b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/RowSerde.java @@ -32,6 +32,7 @@ import io.delta.kernel.types.ByteType; import io.delta.kernel.types.DataType; import io.delta.kernel.types.DateType; +import io.delta.kernel.types.DecimalType; import io.delta.kernel.types.DoubleType; import io.delta.kernel.types.FloatType; import io.delta.kernel.types.IntegerType; @@ -126,6 +127,8 @@ public static Map convertRowToJsonObject(Row row) value = row.getFloat(fieldId); } else if (fieldType instanceof DoubleType) { value = row.getDouble(fieldId); + } else if (fieldType instanceof DecimalType) { + value = row.getDecimal(fieldId); } else if (fieldType instanceof DateType) { value = DeltaTimeUtils.getSecondsFromDate(row.getInt(fieldId)); } else if (fieldType instanceof TimestampType) { diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java index 7fdffc03041c..81edba927770 100644 --- a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java +++ b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java @@ -28,6 +28,7 @@ import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.segment.AutoTypeColumnSchema; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -66,35 +67,35 @@ public class ComplexTypesDeltaTable "id", 0L, "array_info", ImmutableList.of(0, 1, 2, 3), "struct_info", ImmutableMap.of("id", 0L, "name", "0"), - "nested_struct_info", ImmutableMap.of("id", 0L, "name", "0", "nested", ImmutableMap.of("nested_int", 0, "nested_double", 1.0)), + "nested_struct_info", ImmutableMap.of("id", 0L, "name", "0", "nested", ImmutableMap.of("nested_int", 0, "nested_double", 1.0, "nested_decimal", BigDecimal.valueOf(0.23))), "map_info", ImmutableMap.of("key1", 1.0f, "key2", 1.0f) ), ImmutableMap.of( "id", 1L, "array_info", ImmutableList.of(1, 2, 3, 4), "struct_info", ImmutableMap.of("id", 1L, "name", "1"), - "nested_struct_info", ImmutableMap.of("id", 1L, "name", "1", "nested", ImmutableMap.of("nested_int", 1, "nested_double", 2.0)), + "nested_struct_info", ImmutableMap.of("id", 1L, "name", "1", "nested", ImmutableMap.of("nested_int", 1, "nested_double", 2.0, "nested_decimal", BigDecimal.valueOf(1.23))), "map_info", ImmutableMap.of("key1", 2.0f, "key2", 2.0f) ), ImmutableMap.of( "id", 2L, "array_info", ImmutableList.of(2, 3, 4, 5), "struct_info", ImmutableMap.of("id", 2L, "name", "2"), - "nested_struct_info", ImmutableMap.of("id", 2L, "name", "2", "nested", ImmutableMap.of("nested_int", 2, "nested_double", 3.0)), + "nested_struct_info", ImmutableMap.of("id", 2L, "name", "2", "nested", ImmutableMap.of("nested_int", 2, "nested_double", 3.0, "nested_decimal", BigDecimal.valueOf(2.23))), "map_info", ImmutableMap.of("key1", 3.0f, "key2", 3.0f) ), ImmutableMap.of( "id", 3L, "array_info", ImmutableList.of(3, 4, 5, 6), "struct_info", ImmutableMap.of("id", 3L, "name", "3"), - "nested_struct_info", ImmutableMap.of("id", 3L, "name", "3", "nested", ImmutableMap.of("nested_int", 3, "nested_double", 4.0)), + "nested_struct_info", ImmutableMap.of("id", 3L, "name", "3", "nested", ImmutableMap.of("nested_int", 3, "nested_double", 4.0, "nested_decimal", BigDecimal.valueOf(3.23))), "map_info", ImmutableMap.of("key1", 4.0f, "key2", 4.0f) ), ImmutableMap.of( "id", 4L, "array_info", ImmutableList.of(4, 5, 6, 7), "struct_info", ImmutableMap.of("id", 4L, "name", "4"), - "nested_struct_info", ImmutableMap.of("id", 4L, "name", "4", "nested", ImmutableMap.of("nested_int", 4, "nested_double", 5.0)), + "nested_struct_info", ImmutableMap.of("id", 4L, "name", "4", "nested", ImmutableMap.of("nested_int", 4, "nested_double", 5.0, "nested_decimal", BigDecimal.valueOf(4.23))), "map_info", ImmutableMap.of("key1", 5.0f, "key2", 5.0f) ) ) diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..92c3e2d8fba86312278446afc9dfb1bd36a804c2 GIT binary patch literal 24 gcmYc;N@ieSU}9M2JC9}U{JFE*12=Cnkn89H0AEcA@&Et; literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc deleted file mode 100644 index a56f68f447baee7e6115aad6e4ccb68586175fde..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20 bcmYc;N@ieSU}8vExl-+maj;$ag?ZBfJy!=u diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc deleted file mode 100644 index 6b7e86bcf54a017d8701a0f47f79e34e0d195fa8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36 scmYc;N@ieSU}AWpcyWrDl6iFUzYoe4`yQ$3YabQ){QJ_}xX|U#0P?jD@c;k- diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..7bba84c69e67571f0828202bee1f4b6caec2ca4b GIT binary patch literal 40 wcmYc;N@ieSU}De}JE{@5(JZB1cVj}v@ntIFv%c;~T6RRCeEWY%Cnt`j020{`h5!Hn literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..4b77b095336afbd0501e08619c5a99f42dec561e GIT binary patch literal 40 wcmYc;N@ieSU}BJ9eR^E!hfi-(;pCmO4xWinO60KNS;j23gH5RsN})jm_CH|1-cb6)o~0QV*jApigX diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc deleted file mode 100644 index cd8fc7a087f660653d9af8d14911dbbe31a7a752..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36 scmYc;N@ieSU}8|*n37}aTBY}wL1<<+%e~pmEFZ4vvu|kVyLzV@0Ll3cd;kCd diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..d464223f06fe6157db0e725ae07439ca57fc4514 GIT binary patch literal 40 wcmYc;N@ieSU}EU0di892?6>Rk!ja#%{Y%$%Fj9D0qA$buX^N5mr6Xlh08A?p4FCWD literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc deleted file mode 100644 index 038082a933b714e530c2c4cea6b49f5b391f4589..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36 scmYc;N@ieSU}A{lNPTX?c*${h!bFyh?NT>Z1^zgjZtQX4UYOoR0N*?gv;Y7A diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..92345f4f98a29401a25dcba633750cb3b7216aae GIT binary patch literal 40 wcmYc;N@ieSU}CtVDKc-Oe%IQB&fSap7ATkBO{^B>S;j23gH5R4 Ouvlt+3pUnTzaSBFco&EO literal 52 zcmV-40L%Yla$^7h00ICC87fl@eX@`q3>oWVEG+?7lFEsb7v-zj%vt*E#AO1vFwe6v KP0fF9Q|IN#5*5_| diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json index 84803f9483ca..5383b51e0b72 100644 --- a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json +++ b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json @@ -1,8 +1,8 @@ -{"commitInfo":{"timestamp":1723511561738,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"6","numOutputRows":"5","numOutputBytes":"17937"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"b9eae5f4-d55b-4c38-b365-8228ec09248e"}} -{"metaData":{"id":"ce998219-9bde-4831-b78c-14b11f919fbe","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_info\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"nested_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_info\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"float\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1723511559184}} +{"commitInfo":{"timestamp":1729202194177,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"6","numOutputRows":"5","numOutputBytes":"19553"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"ff724e89-210e-47dd-a31f-edec6f2c6f6f"}} +{"metaData":{"id":"68614ef0-e5ca-4caa-bbd9-b3427eecadc9","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_info\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"nested_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_decimal\",\"type\":\"decimal(4,2)\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_info\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"float\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1729202191427}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} -{"add":{"path":"part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet","partitionValues":{},"size":3288,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0}}},\"maxValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} -{"add":{"path":"part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet","partitionValues":{},"size":3291,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0}}},\"maxValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} -{"add":{"path":"part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet","partitionValues":{},"size":3289,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0}}},\"maxValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} -{"add":{"path":"part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet","partitionValues":{},"size":3290,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0}}},\"maxValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} -{"add":{"path":"part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet","partitionValues":{},"size":3291,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0}}},\"maxValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet","partitionValues":{},"size":3588,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0,\"nested_decimal\":0.23}}},\"maxValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0,\"nested_decimal\":0.23}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0,\"nested_decimal\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet","partitionValues":{},"size":3591,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0,\"nested_decimal\":1.23}}},\"maxValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0,\"nested_decimal\":1.23}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0,\"nested_decimal\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet","partitionValues":{},"size":3589,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0,\"nested_decimal\":2.23}}},\"maxValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0,\"nested_decimal\":2.23}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0,\"nested_decimal\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet","partitionValues":{},"size":3590,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0,\"nested_decimal\":3.23}}},\"maxValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0,\"nested_decimal\":3.23}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0,\"nested_decimal\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet","partitionValues":{},"size":3591,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0,\"nested_decimal\":4.23}}},\"maxValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0,\"nested_decimal\":4.23}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0,\"nested_decimal\":0}},\"map_info\":0}}"}} diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..fdd3237e37cd039fe59c9c0d0b0e1304405ac0c2 GIT binary patch literal 1604 zcmdT_O^ee&7#?D_)>=I%vrNH587OQahGyHk-Qq>Oh!k1Gn=WOVnbu(^NjD$6l+vH% zKk%n{@J(i#6c)Da!GjP2dEW2m{m9jauO7Bqe_nrTCQBCIOcVd72pDc4Yt`@*W{Hw27@`l41*C3sj#9B?1wqx&HGcjWa*h26p8NH$l-3Ss*%S z@+Nm;kyoKeQo*OYCbh>+YT5*YS{}J!JV*$R8vwj80FSbn5?bly78AYj z{H^n-2W+{m(3&{9!e=*PHN9gh6Bx>9b`W_2h)EvTA=XJDMW?9MY&nEP@xoT8;=mV%C{CWkt7E`{O= z9e-cboJJ5y4zN*EQMiDwFjI;_<#Y^f5DwjXq~rBwH z&1f|g2hQDg6N~Mv{OE0k?<3vlh2HU_fwO^g-reVIZ$ZNwd4BvsqZ&e2@c;M=*jewB literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet deleted file mode 100644 index 7a105c0f74fff6b494898d601cdbf192592f6b05..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1488 zcmd5+&1%~~5MET8%8_){fmJE6I^oDaQDX z`WShSzDN%p{a8vd*v_F)&_(m}eKX(8?DhLEF16af&px)|rHF2)vG*N(L7Rlux)!sP zBtwfeMHGqER0Z>pTC_2_B7_WUtfp0Ls70uSp(mihhrp!qFJcoVp2-tx6%=h@U@Z_n zbjjp(3SzOd0Ad4j`Z%SrKc$hUP()6dn+1YGa@s`0%fcYTWTqh{IrYVITfQ~v<>f!W zh1K+4=mb)vR+q46XelbArwCNfyyO*Wx7t0%!f0WOr9g%3Of35zqF99itM~1p?GSdp z6A|BO8+E-)z}Qjv!}h@lDU4vhPS)J4mcYk6@0|DeT!EG`AAei(R3mgLiRL0qhrt5> zf<$Y9ZSgT`3_ZR@?f64ol%*_tED({@BTWy3MW5B{ND?6 za4ZJ{Dgm|^U)E5rKYGxJr(P_6xiZ~jV!tta4_niv^XC&oJoVS1CYq{qm4|PJW H@h$%YG+oZJ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet deleted file mode 100644 index bb2fb67389d2373731fc431f3a626e6744c61f53..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3288 zcmd5>Ay~vkN-3p?^dd`{W?tK{lcbwW zR!b=qU5ZFm^rCnw*ke!X_8?f)lLxQ%uotC=J;vfe7L;AT_hvHlN<-82515Ae^S$r= ze&73j-}_!V_ws8NMmU2<@i+IMymKLgC3sb^iqORg2_Y1Z!G(XoUWUK#Kiv6E$zYYc zDF{8Uakq={m?A$xu~?o>;3l|((+k+ZZ-4yo83B2u6o;`mbMimnra=sU{PNMgVZkq* z5BwNp;a|og_&NO3C+|O&1;XJRCuEl}_Q%_wf0q<|g5Z50gpMa6t}?FR<1gL?^<;Qc za3qTavfNA|8Srk?H7mGZ1wR$1@x5Q~@1`ptIoRbiLYoYX3AS^zxz503A8e$=1(kSo zBOsmrtfXs@BmW~jVo4_m@nOb=bWLWH1oMDdv4XC}3je^rN6Xkx!yHeB!Gdx^uuL5Y z7H(HXGt&gjTpHgyLsUEqYeag6%;S~d=^z#wwad1)%$UbS1d7uwYoTDNfuVR=l_q?&_@l9sYtEqiUD)>vvn7L}O_#8s3v z*{!ubEW!H0b%?sOP8%NcA8vcCPSx`#$PpAygQMKA>vRmwWEhB~U24#_N1X+6igNZ8 z#h(J@#F-K6dd}$`{ho6KjEZnu-4rNH`F{H)85=4eo z07~#tL?k__IFHJSS#G zT*N`_Z?59KVNA$_A-0NbDTC9#Kqq*3!ns_pl4Rfv>T$wTEb zbltqBHCxN3y=p^4GE}XFIlM zYd;UI8{XO~HOhwH=#0FvRHLrbHp*|T8>}V5P|X2z;N~_P%kb9dxULOd%qT-krFo-= zXq4ACHuB-js6mGaAt4_FbI_n)_rrj0NtbC0?A2z&vulm^EKvD08|UT~y7p<}|74|H zs!%1Ms14TX*7+B7`ZW3+NYuvO( zbBmpt>x?>!6IN;5D%^02i`T~|Xtj7_yx^3s7pjg;t!oyYD3ly$(Y~VZ!H=UezUa#& JLUZsx&0n-Y%g_J- diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8b6fdcfab6471b588d0356972b59c912b3f3da3c GIT binary patch literal 3588 zcmdT{-Af}^6u+5F#xX|i$KKnSk`@^-L`XxTW;Yu`rF1DGB2tPk?IL4l)L|!yn~!Zp z1iKaKgQXu2E$!3N`c&Fd3hUEWw)+S8)XKgTmVH=!C^nS7b=mE?_fBT6RgCuIp<|ej zGw1%!IluF9$LW_|&0>V(cmRKY^YJ^qaV){3f>nh2MkRz$I0PqlfV=>IcizA8n-a$= zQ&SLnPGf3);gBLfMxjuOE?^?Kg3|NYz}G)|@PvTepAW;kFje9QsA=HCANM|7?-BH3 zDNm0=X8vU?f}h1dee&KTSuog>WCrQU_x;i47e90hI-c`f2cgy&_(icE{P_0IlAz~f zJ;SjIxQJWdy$cM>utLEBbrMi_DT-v!yJ9bAz3VXOsW^t$f4#XCD?(a8F2@jBqXsFd zH}0>kP%xQ;_2(Js5{sZ8=+&Q;bPe*z58)O|IzfoNrnZo-$#jt*?yxG9(X~+KALzHu zQtL-yT4TPmz^>pdWfGiagbHFx8i?UCENWA#K+G89+Hs=dDF_|uaWaD!y}O-|FWi*y zgeTpTeCbx$mmY-ixTml!<7w}9BjQVUBR=lF!V$oyhfj~k>Pu$r%51%~XqQbKMOYq` zG|910PST=gt!6III+gh<lU=9Yz!ES7s$I5gm+eY}>i5(ewPvZo7svq= zj)9_FG0XNKnv7ErNqeed*BiDqD=tw^UZU_NP@&9n)I)H_&>` z0g}#XHm$QF|!`k#sJd zLE6v}jrvtL4YU1f({7||)i={+yJ1>p!_>Z3SB%E;l5G?WK7EapG4I%}RW}N+uNZVr z!z;%Ev8U!%D+};cX}Ycnqtz(DoVQa(8__7Ntgfc~mBFszBlrf{I}m%j#%tUS7~%GU zT?2WkT4|V0r9K6wxS7>c+ZDEwqlW*JNEuZ=OJKt3%+ux52juNt|BtwW{ByU@lmahO zr~`kQ5fsHK>#n`kv+X~?;hzKWd`1}l#US8tL2@)TJk$xTW0E^61a!7PP<&7PW`Z{$ zgt_Ilkzau9FhvquIaeefHgOo z&5vX=7p>gfg^^Lal)E^RvGNx(CCjw4=d<=`CU04D=8%31zW+}Cp--a-O~e0xe*)g` B3(Wuk literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c349328307e78d862796891fd35652bb9007ba96 GIT binary patch literal 3591 zcmdT{O=u)V6t3y%>2xNO1a+(EmQg}Oh6#*AW|P^S4TD5nB!m!x$X;{_o#{?8yFD}6 z`B@Diuqa9hImC+~cvuBt2?(;tdYQwjpzF;;K*(W{JxETgNc14G`d(E}cf}2vMD$=2 zda7Q%`rdord-bX__rl9LN@$V}(MR{byVIAX3QRSs6EZNa5JIAn2)ywD`2;P}z0C&) zy-BKbHI0yG46Zg1jc6DDB&f{i@T-bV&r*wi@zb{-YZCTgAqtC8h3mrz!42@@`;TrN z#3a2$M$$t^|5Y$XpQb_3uqeD6__Ka-$`Slhq7^`)#A zV6A}~=pcPR-`?m?QWa(z4cSwWy*Ov=s=Jy)=HT}%=;<^;KmY3O{X`Km19CM1YiJOe zxcT758iJ_;Y_PzoS9k=2z_0m)Vj6lUfzpsFCL@F|tGH|$D$iCZ?y@VAGmS`2f;p{z z$1GZxVPJ~SBseQZ5Nzm`icEsDoKQ)Oq=A@oCW!?QM%DpIGM&ys=)_F2OZ19Ax*MT$ z{%~8N)BbQz38XvGKzb0RQ@%o54HULwfpj-UmwknYnlJ_t{PEOOqOoGvug*70%WlP{ zaYEH$#Zbx)$tydm}{Z{detu~#|~Dl-`oVK3xt{<`h8T&QmFo)}S(_|8)uY{mF0 zVTzeF+bJFD-OLkBH<3E3FojLC*_2)k>pUDWk`C%JltS*|{D9Mhu={=V3~Tq$=Sk`k zbOCdiXl6l522VhTc(A6kWp*WXsVL5i!K)ao@Y5q^GWsuw{(BG2lGNQu;CNb2UMGkd zHU~QYFp75)l(AbnMzO6VX+jWm$PTXwPS;Cp8nlXVeBlp#=fASoA0XO~I<-h@572zN z(?^t%?ED@;9zk~Z6x>F08@=fKPBe7>2y=&H*oht!!#xE3x))lmUcLkEpGOM#oaiDybBf+;^YKJJUrHToB>{C zi)<48J6yk3!qy=l%}b`V?)kjUJ|?boN=#8MqZPhEc08W$hIkx^$K|?Cf>7-YD0q zMrwSuiEcte@Dfbd@}A_&a{%V2gb`m15{?k0$1`U~x}kMVa#w|r&dvu) z;7M?pUS{~iBF-j5S9 I2mb>81!9;8+yDRo literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet deleted file mode 100644 index 641396cb6f7cf670208d1da2d475f6dcc17d490a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3291 zcmd5y^BTX~+z_MJ;B*!U1*hR#u`IQCu2|F~*oud?~@Ky=z;^N^+!C zT4M~k6iV>Hg`P`Vs3|6=6j~@gxU|srQc5ocUlMZiq0k=GTX6DbX0fue`8cVN{rgWlnH zk(EfgtqKOP@7bkEy|wB#Z_l?DSNxhwvWO_d*uYf}m9UX@n@x9ZzFJ>uKo-^M3fNUr z3?-0UGwpDL71O`EX?0^gSO9*^kHPAK@wR; zGyv-3tB7JVXFkkpP^xB0L#oN*LNLX2iq7PZZ9h@z3Wq!KA)6G|V(3X&&8fRDn&SEA&==t|Cg^-~|XQyzs?LMt{c8AN3O(<@Zvt;UjWp zx=u{fS= z(4ugUoKPdO@%sRILa=+MV3*PDY0UVYbYlDobVq&INuT1wZw&O1`_KVUEFw}q0W_*( zseXBsZqUhm2N-yUlRUFdb3D%njO{^&<#|`hk!(-4qlbeNvI5vuAcnxeo{40{Vkm^N zR8H9;GvJpDi{U0*Yr3f8_`oHHj2$kZQ|J<10oBS?s9n|vc{g-q$WL^D+{`kcw&6zA z%wYwN#tD$(d~(na=)DwZ=izEKcpnwtV81j&W`W-}(6XSz@Kd``P~Ii}IN6WGv3@3j5UZZ_VxYkuf@ zZs;11``4{-ZPmBNEz#(#g0)oj1FvO`zqf9&mIzB#55$pL(5NrNtlka+7rK}=4lUI$ zSY1SGe0^i15bsPHbc_%a@-+}g4f8e;u~84*YP~fBRDx#X%)C<9K2H3f zc*;W+M+qpZz2kKA><%5jjsC}eCHWU-pUc$b){M7VNl~1!?vC8F368jVVSXx1&$RSZ zNCMEk;A0!&@`n`=LZ^ueKQ0-5xktjKpy7s~#$|7v`#{{>K*m&b@L#JnhoVj~H0P2&j*N!G9Hu;OBAsyY~G7!D3&ES&TBze1i|~{J5nFI==T*k$%)k(4$!2@9f;` z6Z8VCpFsL`W=xLEzVTMly7@!?AFSC4(Tu9?SHxq01~=p}5Pg$e5u= z#srv)>UYdi>mm#+;iFE1vz=Svu}ql+XBnZPvoa+O#EdgROsN7fV~mRvM8&fZIx-Su z9+&*Pb_CD)w_6gP_HPfQK)M|bq(@Oa1iknYFuvafKcaAOd`@6S!e8!LAG z=0c;m>{e{7AuJC|x>R;hR?;=QUbj~l%GFX0@~q4hz{RYr%U-$B#1b$Bsy(jfR@`cn z>i0F8^;WUTH^?Crj)S6HwJYv0nn_R)Nq@HLHkz)pAZ}64-lFg=P@&AS<|CBBIM%>I zzb|lVtwqn}N&_94TP#;>Z&;(%n|_mCxgTo9Ug<rGmN)rI35*A8;BW@~97A zB6~gfWt5zU&NYUSV&sHmump672Wu)>CZ*(jf%l8fsu--W?%^vD{Ru^X+Jh~WY)1me zQ)2QuLClaj(D{c^yc5TSe5%qYwxk4(bApc9VKu?%dW}qjRsq@EMvb(3?D{AiDUh4dIG<5zHbH`)Yj-C<2Hx%?iFOA^{Kqbyd z?i5hJ)P)jfEL|g0uoLo32UM8na3wPKh+6xlm*#m-PGGI8wd4K64AKJFlYv992Oo7Z zyrUeRl!^`hl#c((I6gISONs_5Cij^+g}JPd0mqlvK%1&m}k&(z;%&FIo1A4O7}; z1KYao)*IzoRgaBZS!)#OBPTQ(w>%4G`>mGSwCc6Dtcu&T9lL4kU#e?nb9Kcv^Cq9Z zX4)*3UC(Kl`Ri*Yozw8B?10!;^J>*4SXEn|XTxYU^DyV#wAn#4^K0wt>0oDgY6J*@ zLG}#9{;BZ>cLPSayX4kEUaVD{cDdS^1ykJ2`nl~2N6AUU|4F2bs(>XhQSQ#ujq?ZO zpI!ftxWfE%w=a|eKT@cJV4E2h#X0M)bJVlrKg8j$1F*gz41Z$~aJV2fkscfAhSoL7 zT@^w)J1;1KCqXm86A;4O^4lmVK=yLt3rk+jrmQmAs~2vs8|z4)_Q%LemHLoBcZM=n zX57jQr50Od&lz$SCo;M5%;;4oyLe@M!YyX6j*mLIE2Blnb~BeV?!;)$aTe_nqYdwW OXMfR08bWjM58z+WsS6wc literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet deleted file mode 100644 index abee0ea5d0a95f6a5282777b5b7c27e97f96ffa4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3289 zcmd5zI7=M{e#xX{v%f81C@X*xoG`ts*Ti8Ph)s!)}Xp-G7l5|8mu{=hy(i}b7CJKqncsVdwQ zLf+7X+fY2F$bXW=U>2@2eiPo3H>p8iescUojUW#f0!Z?r>jByc}Pk6ld%Wg8_<44aO`J+w?VM?^~WCyy)f73;R zE_aebk$cWYo-2dzarmh;MSs}-W^b?r&0&{QgiLcVzRNFu+ur10GI9Zthl@gFNklgc z?ey0qT~oXIMTS(;86jd?=F_?+i%f*(9;;&Gx)vLcz+Bb8OUv1#FnGnIP9xavFfuK3 zbr7r&Ckz(4Ok;)k^1ahcrSsS$(bH^^mi^IDj4t@YU5U>6!xJg69>fD{Cr)R4heJ7V z*h$be-@31O3}F1@+DxjoZZ+>MwN}=gnnja@%A=AdRcum_w4~K+S{qB1dbxotDsv^o zRgg8gT4}jd!v5jf=Tx1VQ+K)lK+A2mS6pv_9753)9Ob%Ib4JNrnuAE%wYt-C9eXLd zM7em0;w?cr3Ff4aPysnjf??1%*p2pb)!`&+&06njDAD4u z_>oZ+KnXERn53um#~B=MSudzW(YY>EViKEW^BJ`imS{L)BpZ}eXhk+r`Usm4_B=_i zvyMXFCYePXJbe_v#fZ{E-L9%&Q`s6TXBJCdscWJD!6FLZdyAsKpM9iWUo>*KrFmC`ij33Q4_BwWb#9-D<%2`3Lfh+TX~qu%FepQ_X# znG>M-a%az}k!bKIfILU+-YGcbbVn)={vaL(e-7R0JsiX@*uyageI@qL0jT6DQalIL zuj5Gl3YG4#nM?-^BEw;xd8av>=jWX5vwoiELpe>8J=snl4nfEZ;82E!V8EkJha;}x zS!*<9uXq4Y`?-cYxL8wO9R~+4JY@WEflg*+wg%VIP1G*$gQ6SeqmX+kK<*^@p7s@7 z!uqI;qj3(TcRmH^$80+W?@T9=_WG#ziTFiV_1=h{6Soao7CMYLg$JDLJI=*^RG5@e z*$d@5x^7O>8qHPHTDQ;;P0<_82Trq9Y1FmEgjq1hh<4+QN9%soL>s-|c3iXB_{gj| zu4P-UrTyBsX}BBfjxl9;jn2p!<%(0aTgKFfn+9)*n5x(?`)<`neHCx@cC~7uiy2dB zsZP%5A{tYhTU)tcW!Rttgn&@Y!R$Ba5478-pz~m3t@Vu zrKdwEK=*mU5LYRwV9Q;g)z z{Dhew$u74mReQuSdt%Hk-Wpr6Ehm37?@W#rZF||ep+CkS QM;H8%Z~F;Zz&|yA0a8}QDTc+vM_ADMz*iI(ZJuRnw^ zGDZpwgpnZ3tpwq!+_Bel-uE^5sWeGH|Kk3m!7@aLT}~3R#lg5_4}RU+;9#-~He3`c zOCq{qXs5p*>6+S$A@oQkoe?6YWj?KIvdBbW?z1YA*R@DK1andU9xZ1N!(fQl$P|L@ zJ%*-bt`365*R`;jYXY;7#@$1TsdNr|BzlT1&{gm0AVTN8#~q2zc#nsYZ{3Uf)^3ze zdk*`u@30;7!FCmw0f8J{o=&#b%;ufN*3zn7GiicQc}&uzsznNtmN1)5bA7Q|Uv40a z%4`{N6=Y3zs;xGauz$G5ZO5+J^)~k(Y_*%6rMA034xnffj&j|s*<)lj#X%(Pa@}sV zZEG>SL^*qj;x0is3Fd@{Pz5v7uo|6;V{;OT2 zMzkm_USw1SP)y7cCg~~tVOmA0>IIc3I@g6tOky)^F0Gb>5)Gah8TU&nup*l%eS}R2 zdzzqESXZI1lJo)&o<0WPVn}JBZckOPsce<4rWeX?scWJD!6FLZ9fi^VGEN@eh+TY#qu%9c zpQ_X#=|iA-a*v-T9pA%V^o%_`;-E)j4_$uM zxQY}{0rl!QRKG%{TWmVr1%t?NkZ0a$PUiV3XZvh`=XqaF(L`Ui{zWm^$=r@OI~+eh6`#4oz4J0f~U+%{-g=rH0G?s2Yf zIT!n0VNzOUzbRMHbu*gQXs%? z$8k(_F{6Z*YG;fdqEXt|+|2kZg9hy*_=I8(X0JiN<%WT7X|LE#*q0jhwpp#W=0N4r zY@VAp)U{6%|0g>YQu!)@qS{}l+vnfV$=m3EyssetZrB$JHMlkXWkygG=d8P9H*J99 z-n$hMTaxe&GXgdy{B0-obyR z)*SUZ#b_>@%V%?=JEmNjytop~7BV#EXlR9;63Bzwf>I^J+`eiXKec$-HmA z_xpYS-*o1+H*%EFBpsl49v|HcC#eL>C{+mQ9hC?n;ZPW^_yPGC{OoW4{&`Q5Doia( z$V(bi>kWsZ@}DFW%gCS`fj@YLdtRmn{r2#uyS)N*UlBpW5)rU9Fyi3H&wh;wdWnpy z7XtPCzYV6+7wHE(_a8?E#_lv@3^U7I!uPh0_9B9gYrSWC|2XI&*3*xFyBinue5`9= zKIn|m5v{!QO_Xy}IG(kW4bo+&5a7!g!C4@be@v*MSEL(xN!>&+X*Ft##=A!x@ zEwYO+u!K+lI2h~Q3Xf%sBpAyA8W<~M&_E1dBce8j3dAg8Tsx^MbQ)YIdQzRGOYYsH z5S?*vw(S)c^y6Gi#H%4D2 zsafb;eUL=;qL2)h059;srl?EmQfjuu`$c0_3@ogB_)0`SK=h*+&63okkZ*WeOrGb& zlsW@C|Imv&3971JDCotOl%xqBpkq9&CRn&$S0_QM1nmpIU_1L3t^N?vepaYKQu~1B zhC6*q8HvX41LP^e?wo?Jk?#F2G=2w;AJ6zx=#Kla6F$R-?-BI7F7)9L&omy9;whkR zsS72}qI6xIfSr(M98f`?gOwPuC&>0o7v_0aPSSWsw&VT72(bWmWr!j0!KZ&bxWgEp zl!^u%q5=2f7{g6GsX7|)w`TQS-i(0DI9>S_OFLMJj2k_`xEDv4@oY@+&hcA3Xlra| zfS1(;Xq>%B=`uj#oMzKHCqjONkn7zD`3#O^MIV%*8Bc-a6M=!gtFDJYJGC24@;T41 zXg5xMz9M>t9g;9dVeGNK{|33fLN0Y9rb;PAeGt0@r$R>4>W#&$xopCe&a#1>y=^y| zm3mE!PBV*J~}aQfp2F6{lG{H?MG%oFx8FJY}KsqXZO{&T+bNeuvz%>;K_b zkbiFWg;L;p3U%ObGlHTxXWe;6Jv;sb9R4~0>kGp0HwFQR3(}*Rk)cj#9h2NqAqc1U zg5q1^HxoPoAj#7X#bDt-HxamSSlrA|g_Xml9-jM%`p~*X+)w z6%lgip`{)?)elbCM{6())gMR$;!%zET*!|@sCXiXMqj2Me*XKjbd0fYSTH6$Oho1`s}e$c)EfbrnMM5eW3HE+zd7M7iwMbm`JLy{&{ZBml7wAE}{ zt8>-*Vgp%JrYeZ5Bx|x;ZFy9}{^8o=xK7Qfd)&XTn%f}Ezoa4s;|jrP3ja1ymT*M&+jE9l^P`X2xx)a!^hM}Jora|JVxy9DcI$7dnylp2mQVi{4sQg_pp;ZVGloX z(C1fJi_q|-hjTF~V)Ny#=qC>_H7wBZR$d=(+xrEx~eNc46 zLLBl=29O(RzNc*k*RVb$<7gZM>7P#l`T@J0fOl>`mG%3m|B3iTSM}eBo)@nqO1zEPNzQ`sNN0J?5o(;Ces(^|365KYk=&09{hRc+L@)Tmi9M~HUuh)3(D zYod+bY&)LWY`kyQ9M7^X&(a?EtQp?wiero!exoz;#$wfR?Uph2-kQN%BBrV~%z>NR zs4wBI-gaFJUCbCmOLg)_2hkW?TVKzIE29P-B7}ru4(6aizw3vAZs{yJP1qM2b89>R%;Fhono+H7Dmm&;P8CA>e_?$e6dg-EsR{TOY@gUi_Svn%IJt)zC5yETTbCp c!6}ZEZF}CjsNcsQM<@J{&w2@&!M`D!TqKk(Jd#bx9JroZ;^x$Ffw5&^EK^E5Udv7u`wWVoA4~BH| z-uJ%u`+fi3_og$iy_uneCg=dY^T#hgNC_&zRG}&%y;+G65)R4m#1F{FXn{W6-}y0| zpek2W2zg24YQ5o*a_JdCJqe>1cu6yY|r(p1Q7z&sO{Kg zyV5}Y?s}uvEH=aj*@vQWP?RfX*&ZZQ2?UX}=PP!-VOw*-Ey~GT6tM*=lrzU%gc6X` z7%=p>0;}4bcWj~5+m_kIQrUC{V_3bZckn6fP|N0W+e2w8B|PkcoXuS~ou&=d4b~Ga z3=-FPssdMpuM#Hd3B8q6q2BczQB)m6M+tpXfh_=4^HH)!?e zi1t9G21)J#THESl$|z|39zY%w?9M5;jdUAbX#6%@EN$bDp*!rucK8Gz?jh(-7y57j zppuA4{uofV)B`2XqjZT)z)r|BHYh*O{wx@=ePsK+3-jEP6ExP5?eO{GgjfJA8Da>0 z@aZ4-M;ODS(x3qcXu!=F#&9!o8h8J8SUs0FBj7T2SMkKs_GdxHk9&Y|H-;|n0%Kg9 z9Pfzx#~P0_z{_lbErDv`JV0Wf=G{6Igxo^N_q!4Dp#pCh`k)Ml@fb+a5jg03Y&`_p z$(E83eO^4#ZtVJEM)VZFB%zN&+vDf{4sv~qTx=uCq@>Eei}u5wo6x1x zykV!W+qHVBTG1k->0Ek*Xv0S|>erk!^!95_yOFL{-%Xe8hH04%Q~O3;H5x0+wlQXi z?rWrsg_7-9bz|)9RRjApER`$}yJ}9gvIw(E({W5_t;QJic{^pa5sk6cwY8MD)4w%5 z1douff!Mt@-V$a&3%3{T8pw;)O2aHw>eE0aXx2{68@Ni268|Tj@=$qE0*X@SI9)!y zL+;)6|M1JtzcBkuDR4c7I`Fm`eo>sV?%G#9zy5t3@j3wWGs1{Bh5(Kb3}sX2hC88k zOmasBKb-9s6weaxFu@84p>Mgz$SXkpa^eaXoT`aY<+9h$e7r8N5pB|KBd?Te18(0L z$fPr)>CC{;e6!?O1J-;tlON5DT(olY7e=#oF?Vrv#L8b7DO#qTIiIn!BYDf3H;45W Ry#Jm2C7;CznSp-+{{rFn4{iVe literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py b/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py index ffca055e73c1..52cd3ed3de55 100755 --- a/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py +++ b/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py @@ -17,10 +17,10 @@ import argparse from enum import Enum - +from decimal import Decimal from delta import * import pyspark -from pyspark.sql.types import MapType, StructType, StructField, ShortType, StringType, TimestampType, LongType, IntegerType, DoubleType, FloatType, DateType, BooleanType, ArrayType +from pyspark.sql.types import MapType, StructType, StructField, ShortType, StringType, TimestampType, LongType, IntegerType, DoubleType, FloatType, DateType, BooleanType, ArrayType, DecimalType from pyspark.sql.functions import expr from datetime import datetime, timedelta import random @@ -73,6 +73,7 @@ def create_dataset_with_complex_types(num_records): StructField("nested", StructType([ StructField("nested_int", IntegerType(), False), StructField("nested_double", DoubleType(), True), + StructField("nested_decimal", DecimalType(4, 2), True), ])) ])), StructField("map_info", MapType(StringType(), FloatType())) @@ -85,7 +86,7 @@ def create_dataset_with_complex_types(num_records): idx, (idx, idx + 1, idx + 2, idx + 3), (idx, f"{idx}"), - (idx, f"{idx}", (idx, idx + 1.0)), + (idx, f"{idx}", (idx, idx + 1.0, Decimal(idx + 0.23))), {"key1": idx + 1.0, "key2": idx + 1.0} ) data.append(record)