From 68a26b47b29d3308ed36026bb4d24c9efd3099a0 Mon Sep 17 00:00:00 2001 From: dushayntAW <158567391+dushayntAW@users.noreply.github.com> Date: Wed, 6 Mar 2024 19:22:01 +0530 Subject: [PATCH] fix(ingest/delta-lake): support parsing nested types correctly (#9862) --- .../ingestion/source/delta_lake/source.py | 69 +- .../delta_lake_minio_mces_golden.json | 108 +- .../local/golden_mces_allow_table.json | 439 +++- .../local/golden_mces_inner_table.json | 398 ++- .../local/golden_mces_relative_path.json | 52 +- .../local/golden_mces_single_table.json | 176 +- ...den_mces_tables_with_nested_datatypes.json | 2231 +++++++++++++++++ .../local/tables_with_nested_datatypes.json | 11 + ...-8406-cec97e7e5d47-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-8cda-9b17f7c45c20-c000.snappy.parquet.crc | Bin 0 -> 20 bytes ...-8b51-0eeb441092eb-c000.snappy.parquet.crc | Bin 0 -> 20 bytes .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 24 bytes .../_delta_log/00000000000000000000.json | 5 + ...49db-8406-cec97e7e5d47-c000.snappy.parquet | Bin 0 -> 713 bytes ...46f7-8cda-9b17f7c45c20-c000.snappy.parquet | Bin 0 -> 1380 bytes ...4986-8b51-0eeb441092eb-c000.snappy.parquet | Bin 0 -> 1380 bytes ...-a021-4198a3fdd2a9-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-93c4-917f41926d19-c000.snappy.parquet.crc | Bin 0 -> 20 bytes ...-9b42-1df4d0e934c6-c000.snappy.parquet.crc | Bin 0 -> 20 bytes .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 24 bytes .../_delta_log/00000000000000000000.json | 5 + ...45bf-a021-4198a3fdd2a9-c000.snappy.parquet | Bin 0 -> 713 bytes ...4fdd-93c4-917f41926d19-c000.snappy.parquet | Bin 0 -> 1380 bytes ...49ed-9b42-1df4d0e934c6-c000.snappy.parquet | Bin 0 -> 1380 bytes ...-b331-a5ceb2b48436-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-943e-254432f372a2-c000.snappy.parquet.crc | Bin 0 -> 24 bytes ...-b154-65bc9d495f79-c000.snappy.parquet.crc | Bin 0 -> 24 bytes .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 28 bytes .../_delta_log/00000000000000000000.json | 5 + ...4db6-b331-a5ceb2b48436-c000.snappy.parquet | Bin 0 -> 872 bytes ...40fb-943e-254432f372a2-c000.snappy.parquet | Bin 0 -> 1710 bytes ...4bfb-b154-65bc9d495f79-c000.snappy.parquet | Bin 0 -> 1710 bytes ...-8d1a-3b0e4fb3db80-c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b50c-b353c7380ad0-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-804e-322d552a034e-c000.snappy.parquet.crc | Bin 0 -> 16 bytes .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 20 bytes .../_delta_log/00000000000000000000.json | 5 + ...4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet | Bin 0 -> 469 bytes ...4072-b50c-b353c7380ad0-c000.snappy.parquet | Bin 0 -> 779 bytes ...4b3e-804e-322d552a034e-c000.snappy.parquet | Bin 0 -> 779 bytes ...-baf1-57414786faa2-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-a13b-9f7cf9560ad2-c000.snappy.parquet.crc | Bin 0 -> 20 bytes ...-8007-bcb36dc0a8f6-c000.snappy.parquet.crc | Bin 0 -> 20 bytes .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 24 bytes .../_delta_log/00000000000000000000.json | 5 + ...486a-baf1-57414786faa2-c000.snappy.parquet | Bin 0 -> 641 bytes ...4402-a13b-9f7cf9560ad2-c000.snappy.parquet | Bin 0 -> 1149 bytes ...4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet | Bin 0 -> 1153 bytes ...-bc4a-f4daa12b85cb-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-89dd-0a2e03f8d95b-c000.snappy.parquet.crc | Bin 0 -> 20 bytes ...-b22e-60c196d352c1-c000.snappy.parquet.crc | Bin 0 -> 20 bytes .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 24 bytes .../_delta_log/00000000000000000000.json | 5 + ...49cb-bc4a-f4daa12b85cb-c000.snappy.parquet | Bin 0 -> 625 bytes ...4b14-89dd-0a2e03f8d95b-c000.snappy.parquet | Bin 0 -> 1074 bytes ...4d2b-b22e-60c196d352c1-c000.snappy.parquet | Bin 0 -> 1074 bytes 56 files changed, 3351 insertions(+), 163 deletions(-) create mode 100644 metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_tables_with_nested_datatypes.json create mode 100644 metadata-ingestion/tests/integration/delta_lake/sources/local/tables_with_nested_datatypes.json create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00000-2d6d7c61-bb5a-49db-8406-cec97e7e5d47-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/_delta_log/.00000000000000000000.json.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/_delta_log/00000000000000000000.json create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00000-2d6d7c61-bb5a-49db-8406-cec97e7e5d47-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00000-4dfde8c6-b3c0-45bf-a021-4198a3fdd2a9-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/.00000000000000000000.json.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/00000000000000000000.json create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00000-4dfde8c6-b3c0-45bf-a021-4198a3fdd2a9-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/.00000000000000000000.json.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/00000000000000000000.json create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00007-7f5b112b-1800-4b3e-804e-322d552a034e-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/_delta_log/.00000000000000000000.json.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/_delta_log/00000000000000000000.json create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00007-7f5b112b-1800-4b3e-804e-322d552a034e-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/.00000000000000000000.json.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/00000000000000000000.json create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/.00000000000000000000.json.crc create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/00000000000000000000.json create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet create mode 100644 metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py index c4d01be52ae7d9..977282303e37c0 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py @@ -1,7 +1,8 @@ +import json import logging import os import time -from typing import Dict, Iterable, List +from typing import Any, Dict, Iterable, List from urllib.parse import urlparse from deltalake import DeltaTable @@ -35,23 +36,22 @@ read_delta_table, ) from datahub.ingestion.source.delta_lake.report import DeltaLakeSourceReport -from datahub.ingestion.source.schema_inference.csv_tsv import tableschema_type_map +from datahub.metadata._schema_classes import SchemaFieldClass from datahub.metadata.com.linkedin.pegasus2avro.common import Status from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.com.linkedin.pegasus2avro.schema import ( SchemaField, - SchemaFieldDataType, SchemaMetadata, ) from datahub.metadata.schema_classes import ( DatasetPropertiesClass, - NullTypeClass, OperationClass, OperationTypeClass, OtherSchemaClass, ) from datahub.telemetry import telemetry +from datahub.utilities.hive_schema_to_avro import get_schema_fields_for_hive_column logging.getLogger("py4j").setLevel(logging.ERROR) logger: logging.Logger = logging.getLogger(__name__) @@ -126,26 +126,57 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> "Source": config = DeltaLakeSourceConfig.parse_obj(config_dict) return cls(config, ctx) + def delta_type_to_hive_type(self, field_type: Any) -> str: + if isinstance(field_type, str): + """ + return the field type + """ + return field_type + else: + if field_type.get("type") == "array": + """ + if array is of complex type, recursively parse the + fields and create the native datatype + """ + return ( + "array<" + + self.delta_type_to_hive_type(field_type.get("elementType")) + + ">" + ) + elif field_type.get("type") == "struct": + parsed_struct = "" + for field in field_type.get("fields"): + """ + if field is of complex type, recursively parse + and create the native datatype + """ + parsed_struct += ( + "{0}:{1}".format( + field.get("name"), + self.delta_type_to_hive_type(field.get("type")), + ) + + "," + ) + return "struct<" + parsed_struct.rstrip(",") + ">" + return "" + + def _parse_datatype(self, raw_field_json_str: str) -> List[SchemaFieldClass]: + raw_field_json = json.loads(raw_field_json_str) + + # get the parent field name and type + field_name = raw_field_json.get("name") + field_type = self.delta_type_to_hive_type(raw_field_json.get("type")) + + return get_schema_fields_for_hive_column(field_name, field_type) + def get_fields(self, delta_table: DeltaTable) -> List[SchemaField]: fields: List[SchemaField] = [] for raw_field in delta_table.schema().fields: - field = SchemaField( - fieldPath=raw_field.name, - type=SchemaFieldDataType( - tableschema_type_map.get(raw_field.type.type, NullTypeClass)() - ), - nativeDataType=raw_field.type.type, - recursive=False, - nullable=raw_field.nullable, - description=str(raw_field.metadata), - isPartitioningKey=True - if raw_field.name in delta_table.metadata().partition_columns - else False, - ) - fields.append(field) - fields = sorted(fields, key=lambda f: f.fieldPath) + parsed_data_list = self._parse_datatype(raw_field.to_json()) + fields = fields + parsed_data_list + fields = sorted(fields, key=lambda f: f.fieldPath) return fields def _create_operation_aspect_wu( diff --git a/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json b/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json index ed65d740377967..9c731fe9a1ba5f 100644 --- a/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json +++ b/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json @@ -45,23 +45,21 @@ }, "fields": [ { - "fieldPath": "customer", + "fieldPath": "[version=2.0].[type=float].total_cost", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "float", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" }, { - "fieldPath": "day", + "fieldPath": "[version=2.0].[type=int].day", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -70,12 +68,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "month", + "fieldPath": "[version=2.0].[type=int].month", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -84,49 +81,46 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "sale_id", + "fieldPath": "[version=2.0].[type=int].year", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "total_cost", + "fieldPath": "[version=2.0].[type=string].customer", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "float", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { - "fieldPath": "year", + "fieldPath": "[version=2.0].[type=string].sale_id", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -380,6 +374,68 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-test-bucket/delta_tables/sales,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd", + "urn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd" + }, + { + "id": "urn:li:container:acebf8bcf966274632d3d2b710ef4947", + "urn": "urn:li:container:acebf8bcf966274632d3d2b710ef4947" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1672531200000, + "runId": "delta-lake-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1672531200000, + "runId": "delta-lake-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acebf8bcf966274632d3d2b710ef4947", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd", + "urn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1672531200000, + "runId": "delta-lake-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-test-bucket/delta_tables/sales,DEV)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json index 6ec6eb2809a105..e71ec172508545 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json @@ -45,9 +45,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -56,12 +55,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -70,12 +68,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -84,7 +81,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -924,23 +921,21 @@ }, "fields": [ { - "fieldPath": "customer", + "fieldPath": "[version=2.0].[type=int].year", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "day", + "fieldPath": "[version=2.0].[type=int].month", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -949,12 +944,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "month", + "fieldPath": "[version=2.0].[type=int].day", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -963,12 +957,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "sale_id", + "fieldPath": "[version=2.0].[type=string].sale_id", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -977,35 +970,33 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { - "fieldPath": "total_cost", + "fieldPath": "[version=2.0].[type=string].customer", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "float", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { - "fieldPath": "year", + "fieldPath": "[version=2.0].[type=float].total_cost", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "float", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" } ] } @@ -1150,9 +1141,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1161,12 +1151,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1175,12 +1164,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1189,7 +1177,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -1455,9 +1443,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1466,12 +1453,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1480,12 +1466,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1494,7 +1479,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -1803,6 +1788,360 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + }, + { + "id": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef", + "urn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a282913be26fceff334523c2be119df1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json index 715beebfe22fb1..11d6924296f77d 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json @@ -45,9 +45,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -56,12 +55,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -70,12 +68,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -84,7 +81,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -889,23 +886,21 @@ }, "fields": [ { - "fieldPath": "customer", + "fieldPath": "[version=2.0].[type=int].year", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "day", + "fieldPath": "[version=2.0].[type=int].month", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -914,12 +909,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "month", + "fieldPath": "[version=2.0].[type=int].day", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -928,12 +922,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "sale_id", + "fieldPath": "[version=2.0].[type=string].sale_id", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -942,35 +935,33 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { - "fieldPath": "total_cost", + "fieldPath": "[version=2.0].[type=string].customer", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "float", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { - "fieldPath": "year", + "fieldPath": "[version=2.0].[type=float].total_cost", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "float", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" } ] } @@ -1111,9 +1102,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1122,12 +1112,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1136,12 +1125,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1150,7 +1138,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -1412,9 +1400,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1423,12 +1410,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1437,12 +1423,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1451,7 +1436,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -1754,6 +1739,319 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + }, + { + "id": "urn:li:container:6bb6dc6de93177210067d00b45b481bb", + "urn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json index 2076ec4096f687..ec1912a7e2c49b 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json @@ -45,9 +45,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -56,12 +55,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -70,12 +68,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -84,7 +81,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -350,6 +347,43 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:85267d161e1a2ffa647cec6c1188549f", + "urn": "urn:li:container:85267d161e1a2ffa647cec6c1188549f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:85267d161e1a2ffa647cec6c1188549f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json index 42e3b19612c2b8..ffb4040d17bff0 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json @@ -44,9 +44,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -55,12 +54,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -69,12 +67,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -83,7 +80,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -686,6 +683,167 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_tables_with_nested_datatypes.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_tables_with_nested_datatypes.json new file mode 100644 index 00000000000000..c8bf54efa46c24 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_tables_with_nested_datatypes.json @@ -0,0 +1,2231 @@ +[ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "2", + "partition_columns": "[]", + "table_creation_time": "1709535902908", + "id": "363c76ca-3357-48c6-b14d-71262be23dbc", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "table_with_nested_struct_1", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "table_with_nested_struct_1", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:string,_2:struct<_1:long,_2:string>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:string,_2:struct<_1:long,_2:string>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=string]._1", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:long,_2:string>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:long,_2:string>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2.[type=null]._1", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2.[type=string]._2", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "delta-lake", + "instance": "my-platform", + "env": "UAT", + "folder_abs_path": "tests" + }, + "name": "tests" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:delta-lake", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "delta-lake", + "instance": "my-platform", + "env": "UAT", + "folder_abs_path": "tests/integration" + }, + "name": "integration" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:delta-lake", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:189046201d696e7810132cfa64dad337" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "delta-lake", + "instance": "my-platform", + "env": "UAT", + "folder_abs_path": "tests/integration/delta_lake" + }, + "name": "delta_lake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:delta-lake", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "delta-lake", + "instance": "my-platform", + "env": "UAT", + "folder_abs_path": "tests/integration/delta_lake/test_data" + }, + "name": "test_data" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:delta-lake", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:1876d057d0ee364677b85427342e2c82" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "delta-lake", + "instance": "my-platform", + "env": "UAT", + "folder_abs_path": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "delta_tables_nested_datatype" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:delta-lake", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "operationType": "CUSTOM", + "customOperationType": "WRITE", + "customProperties": { + "engineInfo": "Apache-Spark/3.5.0 Delta-Lake/3.1.0", + "isBlindAppend": "False", + "isolationLevel": "Serializable", + "operationMetrics_numFiles": "3", + "operationMetrics_numOutputBytes": "3473", + "operationMetrics_numOutputRows": "2", + "operationParameters_mode": "Overwrite", + "operationParameters_partitionBy": "[]", + "txnId": "95fab5d1-7811-4326-96d2-6244adace432", + "version": "0" + }, + "lastUpdatedTimestamp": 1709535906725 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "2", + "partition_columns": "[]", + "table_creation_time": "1709110539186", + "id": "73cae4a6-3988-4337-89ca-af58dd528b0b", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "table_with_nested_struct", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "table_with_nested_struct", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:string,_2:struct<_1:long,_2:string>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:string,_2:struct<_1:long,_2:string>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=string]._1", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:long,_2:string>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:long,_2:string>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2.[type=null]._1", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2.[type=string]._2", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "operationType": "CUSTOM", + "customOperationType": "WRITE", + "customProperties": { + "engineInfo": "Apache-Spark/3.5.0 Delta-Lake/3.1.0", + "isBlindAppend": "False", + "isolationLevel": "Serializable", + "operationMetrics_numFiles": "3", + "operationMetrics_numOutputBytes": "3473", + "operationMetrics_numOutputRows": "2", + "operationParameters_mode": "Overwrite", + "operationParameters_partitionBy": "[]", + "txnId": "e78cea2a-7fd7-4aab-9b50-776e7638fb54", + "version": "0" + }, + "lastUpdatedTimestamp": 1709110542636 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "2", + "partition_columns": "[]", + "table_creation_time": "1708329075098", + "id": "bccc302c-006d-4de9-b572-0c1939e64fc5", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "table_with_string_and_array", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "table_with_string_and_array", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=string].data", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "string" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "operationType": "CUSTOM", + "customOperationType": "WRITE", + "customProperties": { + "engineInfo": "Apache-Spark/3.5.0 Delta-Lake/3.1.0", + "isBlindAppend": "False", + "isolationLevel": "Serializable", + "operationMetrics_numFiles": "3", + "operationMetrics_numOutputBytes": "2027", + "operationMetrics_numOutputRows": "2", + "operationParameters_mode": "Overwrite", + "operationParameters_partitionBy": "[]", + "txnId": "c1497bc8-403a-451f-b651-adeaad02a74d", + "version": "0" + }, + "lastUpdatedTimestamp": 1708329078869 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "2", + "partition_columns": "[]", + "table_creation_time": "1708329893422", + "id": "6f3b01e4-ae1e-415a-979a-34861346809b", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "table_with_string_and_array_of_struct", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "table_with_string_and_array_of_struct", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].data", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].data.[type=int].value", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].data.[type=string].name", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "operationType": "CUSTOM", + "customOperationType": "WRITE", + "customProperties": { + "engineInfo": "Apache-Spark/3.5.0 Delta-Lake/3.1.0", + "isBlindAppend": "False", + "isolationLevel": "Serializable", + "operationMetrics_numFiles": "3", + "operationMetrics_numOutputBytes": "2943", + "operationMetrics_numOutputRows": "2", + "operationParameters_mode": "Overwrite", + "operationParameters_partitionBy": "[]", + "txnId": "dd7ddb85-1b3a-4f18-bd8f-1466adfe66b0", + "version": "0" + }, + "lastUpdatedTimestamp": 1708329897384 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "2", + "partition_columns": "[]", + "table_creation_time": "1709536362734", + "id": "58b63865-38fa-4478-98cf-3a7fd3425e24", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "table_with_nested_struct_2", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "table_with_nested_struct_2", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:struct<_1:long,_2:string>,_2:struct<_1:long,_2:string>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:struct<_1:long,_2:string>,_2:struct<_1:long,_2:string>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._1", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:long,_2:string>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:long,_2:string>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._1.[type=null]._1", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._1.[type=string]._2", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:long,_2:string>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:long,_2:string>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2.[type=null]._1", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2.[type=string]._2", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "operationType": "CUSTOM", + "customOperationType": "WRITE", + "customProperties": { + "engineInfo": "Apache-Spark/3.5.0 Delta-Lake/3.1.0", + "isBlindAppend": "False", + "isolationLevel": "Serializable", + "operationMetrics_numFiles": "3", + "operationMetrics_numOutputBytes": "4292", + "operationMetrics_numOutputRows": "2", + "operationParameters_mode": "Overwrite", + "operationParameters_partitionBy": "[]", + "txnId": "0ead04f5-e706-42f1-bae9-4829f2001edb", + "version": "0" + }, + "lastUpdatedTimestamp": 1709536366367 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "2", + "partition_columns": "[]", + "table_creation_time": "1708330174792", + "id": "9b656138-6370-412a-826b-4f20e659d5c2", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "table_with_string_and_nested_array_of_numbers", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "table_with_string_and_nested_array_of_numbers", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=array].[type=array].[type=null].data", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "array" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "operationType": "CUSTOM", + "customOperationType": "WRITE", + "customProperties": { + "engineInfo": "Apache-Spark/3.5.0 Delta-Lake/3.1.0", + "isBlindAppend": "False", + "isolationLevel": "Serializable", + "operationMetrics_numFiles": "3", + "operationMetrics_numOutputBytes": "2773", + "operationMetrics_numOutputRows": "2", + "operationParameters_mode": "Overwrite", + "operationParameters_partitionBy": "[]", + "txnId": "96c8e2d9-55f7-416d-84de-55d01936e58e", + "version": "0" + }, + "lastUpdatedTimestamp": 1708330178404 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/sources/local/tables_with_nested_datatypes.json b/metadata-ingestion/tests/integration/delta_lake/sources/local/tables_with_nested_datatypes.json new file mode 100644 index 00000000000000..1eb720b9579cd7 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/sources/local/tables_with_nested_datatypes.json @@ -0,0 +1,11 @@ +{ + "type": "delta-lake", + "config": { + "env": "UAT", + "base_path": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype", + "platform_instance": "my-platform", + "table_pattern": { + "allow": ["s*"] + } + } +} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00000-2d6d7c61-bb5a-49db-8406-cec97e7e5d47-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00000-2d6d7c61-bb5a-49db-8406-cec97e7e5d47-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..6910f58465840449f0deb763ccb73b88bda3bd0d GIT binary patch literal 16 XcmYc;N@ieSU}Erbym#QCxYR}fBsB#o literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..397c487f1e597a9bf0e9748588b774761bfbf6fb GIT binary patch literal 20 ccmYc;N@ieSU}E_4b%##Cx6(^CJIOhl>w7Xati#QuDp>hKtN2-@Y-)n(WM)7k;WKb|4gd%ZXmVEf3MIW_ zP){dJouH(RQm-|f0c&D85a#6;vw|rA{$}AOp5t^$6X>{0z*2((63T{dxz9Q2e3$t$#e*rSZ4T1vP&?rll7Rr{=wl0{PKDkvQE zb0=Ewk4LA=YD||N0;8m}>zYQvsMlr3qGUeiRlh(3i+nu(^JN1J0N%D;yi7~GZM#h{ z@iDP4q9&2t=21YFK5|c+Qi*r`eU1PMQYFf z0p2`%5f<5l|AC9Jpl88@H?Lm2h!hduB-13IV7HW+H}k&teXrk3)46-!B#5XaPd>i7 z_;FPw7>-#YA#}Zl5kf*ngmpT8!eAQF$&1$)-=!36#4Io$ZQN+q%n%t%pbmmKiuG($Q(T=Ed`&+g^?i9LO6`TZkzXcG~UTmj2?;1p0> z1&%tFMaK@3jAbSnY*ko=$SN$8ms%ALpjc6rwhlJulsb~K3Zo^MV!B87b*UAZ5v>#z zLu;ZFbPlc}0;8iZMY2ghU}7M>8#5Kdo(B$e+gf?I+JTb0xw=Wyo-x{=m~12c zG@}wR>af9iD=X?Y?LfYjSf50|9}gtCO>3NsgwivC+(!BmG`I^XjY`-)(^!H=IKyaV zcCf)lr!?nh#`#SEJLp@mQkUpCh9^WR$?|x5XxI}Q25E#dHBQ{AKNyeXD^;Uzl##sk z7s5aG3>g3M%nb~9GL4}d*lgPJ8)2>mPbaR{)cjyN>jzp%do*x8$Jd$<=Gw>}vedwV z>jX9!9m07u^E|ufK^_X-lD6X1n)Ah?6xTADxOyEi%Z%Zyr%Q=hmALk@Qr`G7zbYlP zW|I#W|FLWh0Fw9k6mJcuc|P{MX_!^R%olsJf#>9%Ud?P&&GN2O@9k7;Zoj@;Ejx{! ea^JCCbK7)l<%Z+*>@9`;@8d5j!27EM@7F)4t30Lv literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..21c725ed0402f4f97d2fd397e36e33a50115a8f8 GIT binary patch literal 1380 zcmb7E%WB&|6djLMR|~fxkQpQp!3ZoIaE0YZ;uurfT?ra=Srsoz??_`=&_ZovX70>A_nhl{WVP=;ZhzQ-TVuVmSCBQl!J}bk$N;LB3{Q6u-z&7ut84RNgY-g4?aTszNu%EuY zdbyE6Zq!3$8f^oTeK_v>@7G^${R_vG1IG{KO)N_S>w#|s%M_vTIul9C5)Tq7aiJqi zahOO8A+;24{92(RIRs`TE3`w7cyy5>Z63~f2>2Hdvi`^#KQ;Z{iPN`;fJiovCG6UH z)KY+>h9$waf@Q`umKki8S%ttVOoNx2We%WNR^*l@DkLk{keHDfEx{Di1A3^5&1e|W zN>L#kO>~0JVXBCLb@ZJ;w&+((bfk4+rb5^gXohrcdZcxli`gx9pybnB+$3o)8SO7j zc9C|GQiu?BSm(Uu1$B$IA>Uk%ei;FOY{>F9sc|I`N-xvoF4E?p!KaYWD2MGcjX7w9 z9!4u02QxN0B{_dE&Yx)*2mJ`6)I@rPx4`P6Bn`*Mx;3)ENImpaKXb;uI~+(iDtb*X zA!+9?gn#PkVE^gF33PZe^}Z8W%x&pIda4G`M~>Q1{a`%l1!_@!;yRw~tBprfbzt>b z%C+Ho33CU>a2`xN&+2-RheD^QE;!Z3bT%u-wM-_iUPa6_VmRw*Eitnk*RCt&4cGZq z38596d_4Ql$W{O#>43XwBf1Z8^rS;Z#d?+wNLBGW*}hUzCUUR~z21e*nguJC*my!B9dbwq08LK_fdBvi literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/.00000000000000000000.json.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..a3313c598afc99f33d4c5fbe7e22a593385aa25a GIT binary patch literal 24 fcmYc;N@ieSU}6wlR-166rH<`esbAgiGoMWWUr7nn literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000000..85138b9f9efed0 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1709535906725,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"3473"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"95fab5d1-7811-4326-96d2-6244adace432"}} +{"metaData":{"id":"363c76ca-3357-48c6-b14d-71262be23dbc","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"data\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1709535902908}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"add":{"path":"part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet","partitionValues":{},"size":1380,"modificationTime":1709535905678,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"data\":{\"_1\":\"value1\",\"_2\":{\"_1\":20,\"_2\":\"test\"}},\"id\":\"row1\"},\"maxValues\":{\"data\":{\"_1\":\"value1\",\"_2\":{\"_1\":20,\"_2\":\"test\"}},\"id\":\"row1\"},\"nullCount\":{\"data\":{\"_1\":0,\"_2\":{\"_1\":0,\"_2\":0}},\"id\":0}}"}} +{"add":{"path":"part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet","partitionValues":{},"size":1380,"modificationTime":1709535905678,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"data\":{\"_1\":\"value2\",\"_2\":{\"_1\":30,\"_2\":\"test\"}},\"id\":\"row2\"},\"maxValues\":{\"data\":{\"_1\":\"value2\",\"_2\":{\"_1\":30,\"_2\":\"test\"}},\"id\":\"row2\"},\"nullCount\":{\"data\":{\"_1\":0,\"_2\":{\"_1\":0,\"_2\":0}},\"id\":0}}"}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00000-4dfde8c6-b3c0-45bf-a021-4198a3fdd2a9-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00000-4dfde8c6-b3c0-45bf-a021-4198a3fdd2a9-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..3db87037e0232e8e50a9c9c278e13f65af1463e4 GIT binary patch literal 713 zcmbV~OH0E*5XYAoZ@E@B;2|s&EW{<7hfQ?*&$_a6Ya2ljl%dh-mJhG|1s)+Bb40B665W-YL6Yu)h{ z*X0||c}rDO@Z)5I+V_g12sLfpr7g12Q^o~rQ1q}t1SV_C+ zgrJF#1*H|oshAF&P;WL!vmih084qq EUo(Q#+yDRo literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..84638657d66e2387d44829eb991ca74267aa6093 GIT binary patch literal 1380 zcmbW1O^ee&7{{lRSOQjMVP}{?kP>Lh z{D-6v4AU%;5ZbyWAcO>rvM`*Q?#$-obzr}~Uwy+V@>(Mx1+)z$zasAQuje1G{EZWV z@omosP9b^n>DBqq%PH<^4Fe!#I5r(WHsmcVvus#G5)dp?gn}h0_LW%@rj|u+C9_~i zrvqsw@*zqqR3b;9ie!a$NhgYynIv3g$RW=H|MX7Io0!u_J#X;P9-2f%b<9M|^m#~f z10MA|ZwRGzY^1jTcUWu_IjBYc8((426V);tplrSAlyBW<*mCSM(RON^;3K5IpYz#RMqQx8L^!| z2=Bzzq5UT_+t7G$Q<$|j>19+ zZt_CUNN}#}jE-SGnz^pocR>)G?2;N6p*H7>MJX}sRg!oA)yf_Jr_qK*-d|j>iwlAE zHXSS3=Em-b9YkHcF`VY3w&xAqsOm<(*q=GBmACpeqfs@=`&PZbSFPED`hK-+HTKE_ d%e0MM!>*MZmen_RWd3JQZd8EpR|meYzXAToJY4_) literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..783419e5969d7473543d78a3c1166572bfff89ce GIT binary patch literal 1380 zcmbVMU2D@&7(P9TH5b(}*g2d)7$wlqK@CkmwyqS>I~ipZ*`1Y=HgOB4A8XQy)WX0E z|AHT|Ur@Zzdy(Ol;HBV?@M_mXd{4d_P}nqtocFx%ljnKQ`;c_*KQss;D$&U6vx_rP zAsDt7-eBOG2NM6fzl>mKYn|8x|Tw&H3Fo7HUY`62>1Qht1q|yh7$qf z+nx_Nh2+WScNcGNrf|1w7yv?s!=~eB6?q-YEE~3v1O&?zp`awix-v_`R9WOwG7FY; zI-q7EU!t@^C2|0)NLFZinzj%=ICg${M&l?=sLz9T8j+tngJ`YK* zfEn%?1=Lmm4i((1@Vpr2V?ZSWgm}k_wiL^X+}0$8*W$yw$H6jjB=Jwd(zyYRw+hcdKQqu~Qyc crfqB+cCFm7tiHJ=^N&5bQ33v69r(Zg1|TXsO#lD@ literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..247ed10c40e255d6d9ee20e5f55c8114f7351f2f GIT binary patch literal 16 XcmYc;N@ieSU}Er^|1Zl);Pz(#B>V-e literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..c84f84ae758510225847f4dbd4a2f0a5ee1ac253 GIT binary patch literal 24 gcmYc;N@ieSU}AW4^Zlx&`ZH5cYUsXLEmuAb0CG19M*si- literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..da7aa656d8095fc478c07353d98fc79a9631857f GIT binary patch literal 24 gcmYc;N@ieSU}AVbq48u|jo|8&8oDo5%au<90B0}>egFUf literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/.00000000000000000000.json.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..f1ebfa5144737c96b5bb8bb6a3cad81533886d1c GIT binary patch literal 28 kcmYc;N@ieSU}8{Sz1H>a;;M#ul5bs|e_AnLpZD)B0E}=9CjbBd literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000000..d31c179dbb9acc --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1709536366367,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"4292"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"0ead04f5-e706-42f1-bae9-4829f2001edb"}} +{"metaData":{"id":"58b63865-38fa-4478-98cf-3a7fd3425e24","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"data\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1709536362734}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"add":{"path":"part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet","partitionValues":{},"size":1710,"modificationTime":1709536365577,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"data\":{\"_1\":{\"_1\":20,\"_2\":\"test\"},\"_2\":{\"_1\":20,\"_2\":\"test\"}},\"id\":\"row1\"},\"maxValues\":{\"data\":{\"_1\":{\"_1\":20,\"_2\":\"test\"},\"_2\":{\"_1\":20,\"_2\":\"test\"}},\"id\":\"row1\"},\"nullCount\":{\"data\":{\"_1\":{\"_1\":0,\"_2\":0},\"_2\":{\"_1\":0,\"_2\":0}},\"id\":0}}"}} +{"add":{"path":"part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet","partitionValues":{},"size":1710,"modificationTime":1709536365576,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"data\":{\"_1\":{\"_1\":20,\"_2\":\"test\"},\"_2\":{\"_1\":30,\"_2\":\"test\"}},\"id\":\"row2\"},\"maxValues\":{\"data\":{\"_1\":{\"_1\":20,\"_2\":\"test\"},\"_2\":{\"_1\":30,\"_2\":\"test\"}},\"id\":\"row2\"},\"nullCount\":{\"data\":{\"_1\":{\"_1\":0,\"_2\":0},\"_2\":{\"_1\":0,\"_2\":0}},\"id\":0}}"}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7e32df7323c8b7e5a41922f1e595603d4a1432b7 GIT binary patch literal 872 zcmc(e%SyyB6ox~syKXBfxF~@E2Wn_~aXPbeC*opo>qtqPI2O~(^rA@VoA?|)pXp2; z1VQa=ge(p@-^u@`i-bI0!yQt zhS#{vUvSE6+S`QhYK!XEf};hh>avMzWTA(I3s#}UU5hfB_Ng9~i5>Wtf{+}FBavro zS)4UO&`3yiX^UIv(=l4KU1#l`h0b26Zy@=-_P#sh@sUmce-|U#OxXB81n_#8r}MNl zhXFJRCN3uCS=1zgnJn^1;1YYz{AlKSJoe|V&4QU7Gs?*Y;hr5Z7SXA(lI!sGfYlKI HOSxkoxZwoI literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..cbf52efddf12dba6c9c3dbeb5a6c5df6101dd8fe GIT binary patch literal 1710 zcmc&#O^DM#6rN7fC16#AogslBC9-7~YH0f7cFQ7olVw?or-Cab?Zh>h{%n#eQhM>? z&7-G1dFs)VXN3h9yy``~c^6MoM8r2qHVM|h7Q|9AZ)U#vzVE$xX`9z>7zB}vL?iEC ze*J(|f?>KsB!te_1%!|wpj8<7C&In`_UQQR&u}7Oe9QCy0iR0pP0 ztkNd}xrz!cmE`6W%CSOgJG_YS3CzK@gtIi`GtT%%AUY~cGjpcPY17lplCF=lgwls1 zcRd4*A8dn0VtWP}DW|{o{f=|K5V`LYc(Ec*8QUxQM{6h1juGE3Ls6X*KK zx#$=o~$0BK@A_diJ0%XDfP5FClqj4&fa-I`rG&$ntg2 z(|eY0@@|u#i(}1yFtoIW=K1cZ>uW{rwrx4g(;7F&THoyPEe?f^5coF-+QzbQ%^CC$ z;BbHBIA+HIb+Ba>wNuAhV?3D@V`uGhEW{JA|9tk=WEt`Wlm7o%ERd$n7Ox?accOl} z+;j6$1Lh6gsOUz1y)&{MmS>%+QLh-KEmrGnR;pIFwpA&y`ev!iOv|`pSk+RUv5vW+ R?8C2G`im}Q5o*Fe)pzh2du9Lt literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..67dba6c7352323649b495020a55dea6b8d51ce54 GIT binary patch literal 1710 zcmc&#&ui0A9DjXDXAadN>^%|)qeNCVsG&()*Oei7lQBl|RB%$#CT_v>$C6Z$QV>0O z^C*b>19}t$&l7go#hYHln|BdUMMV6*ByED-)b3&_dEfWm=ll75zVCh0)^6O=2qMcw zA)ns<_>5(OVY)yBgcd6tLWtwg5)AAU;XeF)dVKLT91j@Z^85(*`rVU9i>Kk11Gc&N zWRfLEFTcIGkl?#i!a&5On5^4HNIkrn@p^`Pui>@!t+qjURK_B28h#Yw zh6ZbdU%?`Th-895UqRb45C9f8^gjSXylqB!iX~av%?UE$r7~I&q)H$K%uEMUOt)z* zCv3)t8AlYYPLDWVg>vE2p%R4XVVW$_=N!3?a&?(xW)$?PLVYv5i0}!_!8wPsB;zZ_ z_?{yw%8gPpruC%hQ3_m7ZZM9R>)F+$#&McZ`j}^~$Dr|>ZP18qk3l2h^laZB8Rrqt ze4oIJ6>*B#HiIMBqC2n-T&#`4C*z7|Nq=7%*B8b`zrzjX1p17FR+sXk*md_*qh~

GHrAIGeqK6)K6F2ZZ>Mbtfp#3Rm-k6`?h0d&1Olf x6t%*JS#GWuOIEAAQ7o91^+L-uEbW?Rl?oNpY#M9QF8rz`zvxODp&I;C{Q}UndVc@_ literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..6f6b2373dfebab0de54d2ba890dbd009db429a04 GIT binary patch literal 12 TcmYc;N@ieSU}EU`63q$#6GsCZ literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..e8c4254da796f928edcd73de8094adeb883317b8 GIT binary patch literal 16 XcmYc;N@ieSU}7kUm)BW%iAMHq)$ literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000000..0236fbb37d2e81 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1708329078869,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"2027"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"c1497bc8-403a-451f-b651-adeaad02a74d"}} +{"metaData":{"id":"bccc302c-006d-4de9-b572-0c1939e64fc5","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":{\"type\":\"array\",\"elementType\":\"string\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1708329075098}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"add":{"path":"part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet","partitionValues":{},"size":779,"modificationTime":1708329077928,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row1\"},\"maxValues\":{\"id\":\"row1\"},\"nullCount\":{\"id\":0,\"data\":0}}"}} +{"add":{"path":"part-00007-7f5b112b-1800-4b3e-804e-322d552a034e-c000.snappy.parquet","partitionValues":{},"size":779,"modificationTime":1708329077928,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row2\"},\"maxValues\":{\"id\":\"row2\"},\"nullCount\":{\"id\":0,\"data\":0}}"}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5ecd72a84a87ec4941d567fc78eade861d05ac32 GIT binary patch literal 469 zcmah`O-sWt7|u%7ORtp_Je0t|K@AOE-3MO0i3}OyO-0fs&S2VZNve#|AGm*I>S{fB z@R;X)KHfLa^7@|mPVjW&=`~gFFFJqY1@&FvdqW30f(^mq3;>Wi&}Gz6-|>$ZS1sT| zR|;)>=RiH-k{7%*9sd*rUJyZ8sTHPcYF+Vn0RG@gi>ic22~Keg;OGxSZzQ(6Y-(<> zsy=bS4Q-?0ey~GkyXI()v{7|#P=sCumrSGi;|`UyXh(us_?ITim3^1Blr)oeZiPHT zKb~m5YnrInwlTfB52H%at%bV_ANmyz-g#9TDoTB4GYw znM+h{p7wRX5JntjIip+XZ56AzO7arVsqNTa-EvsP?tp38nu}3R`3WD@t^QW))^cbR z%C{a0!--K80Dn)}KL1RZ5B0HTs)1>D3mrx8o1VyP@^e0`kNbzvAVWb0f3sX`mLG(< zz&{X9>lXh=_HhB85R3G}mrIF5`^EhJ8T;!WANTFg zRW9{r#8|8ou0sI4MVbFi)%H zPO5<)wEUpjn5?2WtcH_z&}{|HU=}Sw!}tBGYDe8%)z~k}6B- zU+^y8ym|BFMHH|87f;^2`6kU)!Go8Vy!XEMz3;u5IeqXrV89$!W#8WYe7ox~imu5l zfLm<}06WA$o6cX~#dw$X*yj&le%*8sqG2Y0BMO+gL(Kd4fFO9F%|mU5_HAklLsyp{ zMr{sIgvr1bWLO3qZm|(&z;^f&9|u z6v{Uq3d4y}6aari*&hE&m=EA(egWf6U6>Mi0m@nEaO;?W6Wcr z^4x1qx3^lay=|>-v#s7XlW8iiFkBwMldXge_hg>rGD@-$_So>R|I_gK8pLrq;+{rZWxZ`zU|=! N(syWzM|z6S;5P~Gx%>bC literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..a4eff05371f7c61c587a4c89857e8b1b6389fd2c GIT binary patch literal 16 XcmYc;N@ieSU}A6zn#-?#e6tt;A0PxW literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..b33edc31f6e7f7187a53ed387be803bcfe0f251b GIT binary patch literal 20 bcmYc;N@ieSU}89Ys(QyV9&NKu(UiLYKnw=$ literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..3914d912a54c3034ea65d8752e46c5ac33376ec2 GIT binary patch literal 20 ccmYc;N@ieSU}D&6@$gQ^tRR_R6Bh{q07Rh&v;Y7A literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/.00000000000000000000.json.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..b7ff32e8f6ce97eea21c2d96f2a9d2b55454c843 GIT binary patch literal 24 gcmYc;N@ieSU}9LNbaRdl^RlaJ9h??1r=H~p09#lHHvj+t literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000000..00e2ad37347fbe --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1708329897384,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"2943"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"dd7ddb85-1b3a-4f18-bd8f-1466adfe66b0"}} +{"metaData":{"id":"6f3b01e4-ae1e-415a-979a-34861346809b","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"value\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1708329893422}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"add":{"path":"part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet","partitionValues":{},"size":1149,"modificationTime":1708329896608,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row1\"},\"maxValues\":{\"id\":\"row1\"},\"nullCount\":{\"id\":0,\"data\":0}}"}} +{"add":{"path":"part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet","partitionValues":{},"size":1153,"modificationTime":1708329896608,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row2\"},\"maxValues\":{\"id\":\"row2\"},\"nullCount\":{\"id\":0,\"data\":0}}"}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5d518b0151a55683ceb548b8aa9f51da4d1baddf GIT binary patch literal 641 zcmbVKT}#6-6b+%%m%Uap@L>c74r*xW>OR7dTjOjk= zgD^qJOYXTR=bW26y?qQ^-1@w077MQ5Uy9@nQtmp)ZFVrhQgqPd5+M{};s|cIi`}!y zxGFfrQL->DuCQ}ZkV0mTL?`1Z9H=g~FEv(Nm4Gf#tJ(4h$@QG^1-D{sr^{BL3z=u6 z5weK+h;)B-6idad#bpVG>HLGGV0h(-eBli>>jlseEsQP`Lp}N`L5YHno;S2|Vj(GP z`KL{0b9>K9rFg6?Zi(R01g>TglYyT8wMW*S36!{d(<5G(tbcS76Ff2B-5SLw&^cO+st&~!Vlpd5;CfTuFI@z?ln@WiR z|AZnQJc?(pg2z%19t2O`{42b86yMBl(|}k(lI-rh`M&Rc@4eY+@IB4mC8a0&vgLFk+M{=6!&Y>2LH8<0&Mq z$*3z3R;NDw@Z;kr6d$wLPQg+skT#9?S^U{|iESAwU2&ttW<_{P3u0U)n1~ySjA6wu z!VWX(7V^M$*e;dD(`AY7t49RASf5Fcd2;kL?H-CArv}iOIbxA0oP$jVF)d;lA|L2Xwl%~il&*Y>ofD>K>ziW^KaS=((d zqM+)Pm3MAZ@j@wnCDe!7WeJD=H_5yVt@)VkBNiR2K3lr!%JKDTQvX|-GWJV3X6IOK z&17!~BJH|m^^*gSA9L)xr$^wOiX@Ht=(6FpJRhuuKM?7O^l(}y1Ce=nHoTt5xEhZ2 zoebUV>{z&~Zki>7ZsykAXOWO$>aIQ+x@su66@?hShDGrKuH%7}yd!ZRiK6aaX}YV! z(WpM=sx)KvI2lh8eum-V6zpgVG4Bps+y%AU&MbjVjl4>!fYTyM8FR0FU29XS_VW$}^Hv)epY;~3zP0?+wH2iRR f$?t|-1dD-a`paS1;R|*f->UozFUtVi_{Z}b4+;`j literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..231ba1ba975fbb62d8d6d910dce39e6a0850813c GIT binary patch literal 1153 zcmbVM&2G~`5Z-LuB!>V(>a5n150OQqR0>wyB>htkP%c$fMHQ-g0JUfnZ(9q;No}X7 zDmftW0w693jywSe#E}E^0T3s+^8h>nvl}OkgeVe{?e)%l^UXIiYwz5>?*ah}O7Q8; zuh)4CC~k95Amn<35<;>W1D{d<$r9S{zy^Hz`T4^&3k(g*5mMqBMvY8v=N94-z0vRF zak+OaqQI93tU?VwetYpMI}^ut(>Rp`18?AJs{1B2xuFuK3s94#<}5OsJU=16bQn6C z8;T&sTdV+W3?!z-_F#~{U1Z=uy&~x8W-%U!=<%btdn|hb3@&Pu%@ nyUn$F&0Afmbv+^7Wmh(8tDe^pi)ITytMNsi;g4bme?Gqf-cl29 literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..3c5bb71746af19ebc9a082321b6c96034f65431a GIT binary patch literal 16 XcmYc;N@ieSU}8AeC3`hj+fxDnBv=G~ literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..6b34fbafe876b6d10f2f944ff05d5dd7ea2413c7 GIT binary patch literal 20 bcmYc;N@ieSU}BhiL*~|j)ZG_?#ZnjmL`erD literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..5316ea328f0d9a920e4d97c9af93022c6c67060c GIT binary patch literal 20 bcmYc;N@ieSU}89x@Td4d>h24{Vkry&MqUR| literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/.00000000000000000000.json.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..a895d840fa06c9e44e5c1724e8f8dc31fd5d8481 GIT binary patch literal 24 gcmYc;N@ieSU}8|YI&W#v&OUCoZMV8SZh!a%0AWE2NB{r; literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000000..81732bf30841d9 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1708330178404,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"2773"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"96c8e2d9-55f7-416d-84de-55d01936e58e"}} +{"metaData":{"id":"9b656138-6370-412a-826b-4f20e659d5c2","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"long\",\"containsNull\":true},\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1708330174792}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"add":{"path":"part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet","partitionValues":{},"size":1074,"modificationTime":1708330177598,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row1\"},\"maxValues\":{\"id\":\"row1\"},\"nullCount\":{\"id\":0,\"data\":0}}"}} +{"add":{"path":"part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet","partitionValues":{},"size":1074,"modificationTime":1708330177598,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row2\"},\"maxValues\":{\"id\":\"row2\"},\"nullCount\":{\"id\":0,\"data\":0}}"}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..bd636bb9c2d88367a079b69d0afdff1ec38f9174 GIT binary patch literal 625 zcmbu7-%7(U6vjiSdb8U~3SN}Jz(EZSUELqN@lIsO5bs1JZQ=~3?UtmDG5RLHg70WL zw|OCm;$^cimes$=F%zdV}!Srp9ofZypt1SRi+$q4`;g~;Q~G9UVf zlZ#gHkf)@vK5V@sp+q4{_xiK#d#=$g4~{|52_l3lJtJ(+91YokkyoLOtV(o{kd(v# z^}bb@g(7Z-MJ+7R)shs#vKAS=!!@?6xxiy=tgds5Bm5+V;s%fJ*SKUwYf0|puQFNA z+`FumVwrNj3yKKu{>0;T(?t7h+h(8MH)C3}m6Nx-x!WN8$Hl7JP~}x=nJmqXqx&Rl zBK&*t$Dtk?fUdV|JS%j#-Frw0O$ZH7vsx-1@~lsX35`cQ%?3$d!rssClN?H5WrG+R+qxeV!P^*f3=t7T+}9?~V47}8s*cgK z-t^#cC)rW|1CK)w!UX>S1+U`Oix=NCU3U;gF^7D4e?HIqyh%2%UUw1D#45gj`R@C) ziHyt=8USaS3;+t6PWQO~$VDBua1~#Cdv?s)SV8jFbq%I8U{hrs3Nr@iGe)~e>Dg7v z51@7qUcyfwo_sxPqM$A%q}cgeSz__c=O2*4h;pI$^sRMa8ss@9g`NS94`h- zQz?u||2%AR12-rSq{)vo+{zxQ@?f=+?5Oy`-K2Zp?h=+Lfq2Y%|%{+*tms%$mlHQTq+y&d0Lv63_%bW_W=?gf76 zCDzLAeJfJEEEIU;K1~avZL*_57^+T4aVqp}>!hc(vOgT!liCbsQau4vajf>pJ$?sn zj41q1)C&6}R^5J-svt@>h+WQXXj^~p{Nqq}h9K6)GG6S()v@kX*Kr$;Tb=I=g3zma zou<3gaO=xntFzc>`rX!YqwXy&*1Mka-OH}utS@<9M=eO3zLM-ixmEz!q`$>aYGwU| literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4809a08baac66c10d1060635fe64a5b0ede1ca95 GIT binary patch literal 1074 zcmbtU&ui0Q7=H6LZ8Pi?VejYnNe(5jvOx^Z+PWVU!P^*f3=t7T+}9?~V47}8s*cgK z-t^!_+;KAHg`iaF%V`}29;=S{MC^}36ICRXv?tM4z% zCNeThXaJmTG5{!OI^E;`qXpD)3s>>Qw`a$!jTIz+UDser12$FGp)hTLK5ev%l%8Fs z`~Yg_-~xX7@Z{?`69siCA;m7-$`XrjKmULXMwAQ1r|+$cXF#53Qs^1r_&_E(#O*Di2mG$&QL2+)cXo{hlJySea#l1>P*Io0QXHLibdv zXztQ_J9Fql4}&B{mc`0`==c1HtbY!pPCzI1z?n3qT{BFq$QF^883$yX@io3tGurvc z`JKbuNPyg;iAX~l27o^=;0!-x=s@jAGf|hbC0#xq4KN~o0{44^AO-)T@nelT)NY!m zLW+sHgg=q@)C|zO$Yclb!vb>tk>M5KuW6K;!4KFBt-uzdA8$KqN0I8zXaUYc zKTd*vB#I5E<OK{vH*>t5i8 zUSh4>-nSyv%R+%i?$fj&+9o?1grVw$6sJPpwoZClEBnKtJ*mxLCe;%#6~}6i+~arP z#)!iIM6IwtV%61Xa!b ALjV8( literal 0 HcmV?d00001