diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json index f641706c7661e..a1b3a94ace640 100644 --- a/datahub-web-react/package.json +++ b/datahub-web-react/package.json @@ -132,7 +132,7 @@ "less": "^4.2.0", "prettier": "^2.8.8", "source-map-explorer": "^2.5.2", - "vite": "^4.5.3", + "vite": "^4.5.5", "vite-plugin-babel-macros": "^1.0.6", "vite-plugin-static-copy": "^0.17.0", "vite-plugin-svgr": "^4.1.0", diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock index db55cb4f095c0..558711251d490 100644 --- a/datahub-web-react/yarn.lock +++ b/datahub-web-react/yarn.lock @@ -10610,10 +10610,10 @@ vite-plugin-svgr@^4.1.0: "@svgr/core" "^8.1.0" "@svgr/plugin-jsx" "^8.1.0" -"vite@^3.0.0 || ^4.0.0 || ^5.0.0-0", "vite@^3.1.0 || ^4.0.0 || ^5.0.0-0", vite@^4.5.3: - version "4.5.3" - resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.3.tgz#d88a4529ea58bae97294c7e2e6f0eab39a50fb1a" - integrity sha512-kQL23kMeX92v3ph7IauVkXkikdDRsYMGTVl5KY2E9OY4ONLvkHf04MDTbnfo6NKxZiDLWzVpP5oTa8hQD8U3dg== +"vite@^3.0.0 || ^4.0.0 || ^5.0.0-0", "vite@^3.1.0 || ^4.0.0 || ^5.0.0-0", vite@^4.5.5: + version "4.5.5" + resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.5.tgz#639b9feca5c0a3bfe3c60cb630ef28bf219d742e" + integrity sha512-ifW3Lb2sMdX+WU91s3R0FyQlAyLxOzCSCP37ujw0+r5POeHPwe6udWVIElKQq8gk3t7b8rkmvqC6IHBpCff4GQ== dependencies: esbuild "^0.18.10" postcss "^8.4.27" diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index d8a6e4c6bdca0..9bbeee7cec254 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -38,6 +38,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #11313 - `datahub get` will no longer return a key aspect for entities that don't exist. - #11369 - The default datahub-rest sink mode has been changed to `ASYNC_BATCH`. This requires a server with version 0.14.0+. +- #11214 Container properties aspect will produce an additional field that will require a corresponding upgrade of server. Otherwise server can reject the aspects. ### Potential Downtime diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub.py index 15b50c51a561d..41d855512aa06 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/operators/datahub.py @@ -2,6 +2,8 @@ from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults +from avrogen.dict_wrapper import DictWrapper +from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub_airflow_plugin.hooks.datahub import ( @@ -45,11 +47,13 @@ class DatahubEmitterOperator(DatahubBaseOperator): :type datahub_conn_id: str """ + template_fields = ["metadata"] + # See above for why these mypy type issues are ignored here. @apply_defaults # type: ignore[misc] def __init__( # type: ignore[no-untyped-def] self, - mces: List[MetadataChangeEvent], + mces: List[Union[MetadataChangeEvent, MetadataChangeProposalWrapper]], datahub_conn_id: str, **kwargs, ): @@ -59,5 +63,50 @@ def __init__( # type: ignore[no-untyped-def] ) self.metadata = mces + def _render_template_fields(self, field_value, context, jinja_env): + if isinstance(field_value, DictWrapper): + for key, value in field_value.items(): + setattr( + field_value, + key, + self._render_template_fields(value, context, jinja_env), + ) + elif isinstance(field_value, list): + for item in field_value: + self._render_template_fields(item, context, jinja_env) + elif isinstance(field_value, str): + return super().render_template(field_value, context, jinja_env) + else: + return super().render_template(field_value, context, jinja_env) + return field_value + def execute(self, context): + if context: + jinja_env = self.get_template_env() + + """ + The `_render_template_fields` method is called in the `execute` method to ensure that all template fields + are rendered with the current execution context, which includes runtime variables and other dynamic data, + is only available during the execution of the task. + + The `render_template` method is not overridden because the `_render_template_fields` method is used to + handle the rendering of template fields recursively. + This approach allows for more granular control over how each field is rendered, + especially when dealing with complex data structures like `DictWrapper` and lists. + + By not overriding `render_template`, the code leverages the existing functionality + provided by the base class while adding custom logic for specific cases. + """ + for item in self.metadata: + if isinstance(item, MetadataChangeProposalWrapper): + for key in item.__dict__.keys(): + value = getattr(item, key) + setattr( + item, + key, + self._render_template_fields(value, context, jinja_env), + ) + if isinstance(item, MetadataChangeEvent): + self._render_template_fields(item, context, jinja_env) + self.generic_hook.get_underlying_hook().emit(self.metadata) diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/datahub_emitter_operator_jinja_template_dag.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/datahub_emitter_operator_jinja_template_dag.py new file mode 100644 index 0000000000000..c1b4aa4d7b94f --- /dev/null +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/datahub_emitter_operator_jinja_template_dag.py @@ -0,0 +1,71 @@ +from datetime import datetime, timedelta + +from airflow import DAG +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent +from datahub.metadata.schema_classes import ( + BrowsePathEntryClass, + BrowsePathsV2Class, + DatasetPropertiesClass, + DatasetSnapshotClass, +) + +from datahub_airflow_plugin.operators.datahub import DatahubEmitterOperator + +default_args = { + "owner": "airflow", + "depends_on_past": False, + "start_date": datetime(2023, 1, 1), + "email": ["jdoe@example.com"], + "email_on_failure": False, + "execution_timeout": timedelta(minutes=5), +} + + +with DAG( + "datahub_emitter_operator_jinja_template_dag", + default_args=default_args, + description="An example dag with jinja template", + schedule_interval=None, + tags=["example_tag"], + catchup=False, + default_view="tree", +): + add_custom_properties = DatahubEmitterOperator( + task_id="datahub_emitter_operator_jinja_template_dag_task", + mces=[ + MetadataChangeProposalWrapper( + entityUrn="urn:li:dataset:(urn:li:dataPlatform:hive,datahub.example.mcpw_example,DEV)", + aspect=BrowsePathsV2Class( + path=[BrowsePathEntryClass("mcpw_example {{ ds }}")], + ), + ), + MetadataChangeProposalWrapper( + entityUrn="urn:li:dataset:(urn:li:dataPlatform:hive,datahub.example.mcpw_example_{{ ts_nodash }},DEV)", + aspect=BrowsePathsV2Class( + path=[BrowsePathEntryClass("mcpw_example {{ ds }}")], + ), + ), + MetadataChangeEvent( + proposedSnapshot=DatasetSnapshotClass( + urn="urn:li:dataset:(urn:li:dataPlatform:hive,datahub.example.lineage_example,DEV)", + aspects=[ + DatasetPropertiesClass( + customProperties={"jinjaTemplate": "{{ ds }}"} + ) + ], + ), + ), + MetadataChangeEvent( + proposedSnapshot=DatasetSnapshotClass( + urn="urn:li:dataset:(urn:li:dataPlatform:hive,datahub.example.lineage_example_{{ ts_nodash }},DEV)", + aspects=[ + DatasetPropertiesClass( + customProperties={"jinjaTemplate": "{{ ds }}"} + ) + ], + ), + ), + ], + datahub_conn_id="datahub_file_default", + ) diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_datahub_emitter_operator_jinja_template_dag.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_datahub_emitter_operator_jinja_template_dag.json new file mode 100644 index 0000000000000..7c1bc3131fada --- /dev/null +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_datahub_emitter_operator_jinja_template_dag.json @@ -0,0 +1,457 @@ +[ +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod)", + "changeType": "UPSERT", + "aspectName": "dataFlowInfo", + "aspect": { + "json": { + "customProperties": { + "_access_control": "None", + "catchup": "False", + "description": "'An example dag with jinja template'", + "doc_md": "None", + "fileloc": "", + "is_paused_upon_creation": "None", + "start_date": "None", + "tags": "['example_tag']", + "timezone": "Timezone('UTC')" + }, + "externalUrl": "http://airflow.example.com/tree?dag_id=datahub_emitter_operator_jinja_template_dag", + "name": "datahub_emitter_operator_jinja_template_dag", + "description": "An example dag with jinja template" + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:airflow", + "type": "DEVELOPER", + "source": { + "type": "SERVICE" + } + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:airflow" + } + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:example_tag" + } + ] + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:example_tag", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "datahub_emitter_operator_jinja_template_dag" + } + ] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "depends_on_past": "False", + "email": "['jdoe@example.com']", + "label": "'datahub_emitter_operator_jinja_template_dag_task'", + "execution_timeout": "datetime.timedelta(seconds=300)", + "sla": "None", + "task_id": "'datahub_emitter_operator_jinja_template_dag_task'", + "trigger_rule": "", + "wait_for_downstream": "False", + "downstream_task_ids": "[]", + "inlets": "[]", + "outlets": "[]", + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.18.0/integration/airflow\", \"_schemaURL\": \"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet\", \"unknownItems\": [{\"name\": \"DatahubEmitterOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"execution_timeout\": \"<>\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"operator_class\": \"datahub_airflow_plugin.operators.datahub.DatahubEmitterOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"datahub_emitter_operator_jinja_template_dag_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + }, + "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=datahub_emitter_operator_jinja_template_dag&_flt_3_task_id=datahub_emitter_operator_jinja_template_dag_task", + "name": "datahub_emitter_operator_jinja_template_dag_task", + "type": { + "string": "COMMAND" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [], + "fineGrainedLineages": [] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:airflow", + "type": "DEVELOPER", + "source": { + "type": "SERVICE" + } + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:airflow" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:example_tag" + } + ] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c3af9aae54b864f2542b99a50e0402e3", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "run_id": "manual_run_test", + "duration": "", + "start_date": "", + "end_date": "", + "execution_date": "2023-09-27 21:34:38+00:00", + "try_number": "0", + "max_tries": "0", + "external_executor_id": "None", + "state": "running", + "operator": "DatahubEmitterOperator", + "priority_weight": "1", + "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=datahub_emitter_operator_jinja_template_dag_task&dag_id=datahub_emitter_operator_jinja_template_dag&map_index=-1", + "orchestrator": "airflow", + "dag_id": "datahub_emitter_operator_jinja_template_dag", + "task_id": "datahub_emitter_operator_jinja_template_dag_task" + }, + "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=datahub_emitter_operator_jinja_template_dag_task&dag_id=datahub_emitter_operator_jinja_template_dag&map_index=-1", + "name": "datahub_emitter_operator_jinja_template_dag_datahub_emitter_operator_jinja_template_dag_task_manual_run_test", + "type": "BATCH_AD_HOC", + "created": { + "time": 1726570937652, + "actor": "urn:li:corpuser:datahub" + } + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c3af9aae54b864f2542b99a50e0402e3", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "upstreamInstances": [] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c3af9aae54b864f2542b99a50e0402e3", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1726570937652, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "STARTED", + "attempt": 1 + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,datahub.example.mcpw_example,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "mcpw_example 2023-09-27" + } + ] + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,datahub.example.mcpw_example_20230927T213438,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "mcpw_example 2023-09-27" + } + ] + } + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,datahub.example.lineage_example,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "jinjaTemplate": "2023-09-27" + }, + "tags": [] + } + } + ] + } + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,datahub.example.lineage_example_20230927T213438,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "jinjaTemplate": "2023-09-27" + }, + "tags": [] + } + } + ] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "depends_on_past": "False", + "email": "['jdoe@example.com']", + "label": "'datahub_emitter_operator_jinja_template_dag_task'", + "execution_timeout": "datetime.timedelta(seconds=300)", + "sla": "None", + "task_id": "'datahub_emitter_operator_jinja_template_dag_task'", + "trigger_rule": "", + "wait_for_downstream": "False", + "downstream_task_ids": "[]", + "inlets": "[]", + "outlets": "[]", + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.18.0/integration/airflow\", \"_schemaURL\": \"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet\", \"unknownItems\": [{\"name\": \"DatahubEmitterOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"execution_timeout\": \"<>\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"operator_class\": \"datahub_airflow_plugin.operators.datahub.DatahubEmitterOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"datahub_emitter_operator_jinja_template_dag_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + }, + "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=datahub_emitter_operator_jinja_template_dag&_flt_3_task_id=datahub_emitter_operator_jinja_template_dag_task", + "name": "datahub_emitter_operator_jinja_template_dag_task", + "type": { + "string": "COMMAND" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [], + "fineGrainedLineages": [] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:airflow", + "type": "DEVELOPER", + "source": { + "type": "SERVICE" + } + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:airflow" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:example_tag" + } + ] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c3af9aae54b864f2542b99a50e0402e3", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1726570937811, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "airflow" + } + } + } +} +] \ No newline at end of file diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_datahub_emitter_operator_jinja_template_dag_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_datahub_emitter_operator_jinja_template_dag_no_dag_listener.json new file mode 100644 index 0000000000000..c7b4630b5466a --- /dev/null +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_datahub_emitter_operator_jinja_template_dag_no_dag_listener.json @@ -0,0 +1,457 @@ +[ +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod)", + "changeType": "UPSERT", + "aspectName": "dataFlowInfo", + "aspect": { + "json": { + "customProperties": { + "_access_control": "None", + "catchup": "False", + "description": "'An example dag with jinja template'", + "doc_md": "None", + "fileloc": "", + "is_paused_upon_creation": "None", + "start_date": "None", + "tags": "['example_tag']", + "timezone": "Timezone('UTC')" + }, + "externalUrl": "http://airflow.example.com/tree?dag_id=datahub_emitter_operator_jinja_template_dag", + "name": "datahub_emitter_operator_jinja_template_dag", + "description": "An example dag with jinja template" + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:airflow", + "type": "DEVELOPER", + "source": { + "type": "SERVICE" + } + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:airflow" + } + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:example_tag" + } + ] + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:example_tag", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "datahub_emitter_operator_jinja_template_dag" + } + ] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "depends_on_past": "False", + "email": "['jdoe@example.com']", + "label": "'datahub_emitter_operator_jinja_template_dag_task'", + "execution_timeout": "datetime.timedelta(seconds=300)", + "sla": "None", + "task_id": "'datahub_emitter_operator_jinja_template_dag_task'", + "trigger_rule": "", + "wait_for_downstream": "False", + "downstream_task_ids": "[]", + "inlets": "[]", + "outlets": "[]", + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.18.0/integration/airflow\", \"_schemaURL\": \"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet\", \"unknownItems\": [{\"name\": \"DatahubEmitterOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"execution_timeout\": \"<>\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"mapped\": false, \"operator_class\": \"datahub_airflow_plugin.operators.datahub.DatahubEmitterOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"datahub_emitter_operator_jinja_template_dag_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + }, + "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=datahub_emitter_operator_jinja_template_dag&_flt_3_task_id=datahub_emitter_operator_jinja_template_dag_task", + "name": "datahub_emitter_operator_jinja_template_dag_task", + "type": { + "string": "COMMAND" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [], + "fineGrainedLineages": [] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:airflow", + "type": "DEVELOPER", + "source": { + "type": "SERVICE" + } + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:airflow" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:example_tag" + } + ] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c3af9aae54b864f2542b99a50e0402e3", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "run_id": "manual_run_test", + "duration": "", + "start_date": "", + "end_date": "", + "execution_date": "2023-09-27 21:34:38+00:00", + "try_number": "0", + "max_tries": "0", + "external_executor_id": "None", + "state": "running", + "operator": "DatahubEmitterOperator", + "priority_weight": "1", + "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=datahub_emitter_operator_jinja_template_dag_task&dag_id=datahub_emitter_operator_jinja_template_dag&map_index=-1", + "orchestrator": "airflow", + "dag_id": "datahub_emitter_operator_jinja_template_dag", + "task_id": "datahub_emitter_operator_jinja_template_dag_task" + }, + "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=datahub_emitter_operator_jinja_template_dag_task&dag_id=datahub_emitter_operator_jinja_template_dag&map_index=-1", + "name": "datahub_emitter_operator_jinja_template_dag_datahub_emitter_operator_jinja_template_dag_task_manual_run_test", + "type": "BATCH_AD_HOC", + "created": { + "time": 1726570903946, + "actor": "urn:li:corpuser:datahub" + } + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c3af9aae54b864f2542b99a50e0402e3", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "upstreamInstances": [] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c3af9aae54b864f2542b99a50e0402e3", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1726570903946, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "STARTED", + "attempt": 1 + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,datahub.example.mcpw_example,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "mcpw_example 2023-09-27" + } + ] + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,datahub.example.mcpw_example_20230927T213438,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "mcpw_example 2023-09-27" + } + ] + } + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,datahub.example.lineage_example,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "jinjaTemplate": "2023-09-27" + }, + "tags": [] + } + } + ] + } + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,datahub.example.lineage_example_20230927T213438,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "jinjaTemplate": "2023-09-27" + }, + "tags": [] + } + } + ] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "depends_on_past": "False", + "email": "['jdoe@example.com']", + "label": "'datahub_emitter_operator_jinja_template_dag_task'", + "execution_timeout": "datetime.timedelta(seconds=300)", + "sla": "None", + "task_id": "'datahub_emitter_operator_jinja_template_dag_task'", + "trigger_rule": "", + "wait_for_downstream": "False", + "downstream_task_ids": "[]", + "inlets": "[]", + "outlets": "[]", + "openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.18.0/integration/airflow\", \"_schemaURL\": \"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet\", \"unknownItems\": [{\"name\": \"DatahubEmitterOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"execution_timeout\": \"<>\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"mapped\": false, \"operator_class\": \"datahub_airflow_plugin.operators.datahub.DatahubEmitterOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"datahub_emitter_operator_jinja_template_dag_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}" + }, + "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=datahub_emitter_operator_jinja_template_dag&_flt_3_task_id=datahub_emitter_operator_jinja_template_dag_task", + "name": "datahub_emitter_operator_jinja_template_dag_task", + "type": { + "string": "COMMAND" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [], + "fineGrainedLineages": [] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:airflow", + "type": "DEVELOPER", + "source": { + "type": "SERVICE" + } + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:airflow" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,datahub_emitter_operator_jinja_template_dag,prod),datahub_emitter_operator_jinja_template_dag_task)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:example_tag" + } + ] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:c3af9aae54b864f2542b99a50e0402e3", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1726570904070, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "airflow" + } + } + } +} +] \ No newline at end of file diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py index 784176a3b7f4e..37cd3b792d535 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py @@ -279,6 +279,7 @@ class DagTestCase: DagTestCase("snowflake_operator", success=False, v2_only=True), DagTestCase("sqlite_operator", v2_only=True), DagTestCase("custom_operator_dag", v2_only=True), + DagTestCase("datahub_emitter_operator_jinja_template_dag", v2_only=True), ] diff --git a/metadata-ingestion/docs/sources/gc/gc_recipe.dhub.yml b/metadata-ingestion/docs/sources/gc/gc_recipe.dhub.yml new file mode 100644 index 0000000000000..21734cd4e03fa --- /dev/null +++ b/metadata-ingestion/docs/sources/gc/gc_recipe.dhub.yml @@ -0,0 +1,24 @@ +source: + type: datahub-gc + config: + # Whether to run the recipe in dry-run mode or not + dry_run: false + # Cleanup expired tokens + cleanup_expired_tokens: true + # Whether to truncate elasticsearch indices or not which can be safely truncated + truncate_indices: true + + # Cleanup DataProcess Instances + dataprocess_cleanup: + retention_days: 10 + # Delete empty Data Jobs (if no DataProcessInstance associated with the DataJob) + delete_empty_data_jobs: true + # Delete empty Data Flow (if no DataJob associated with the DataFlow) + delete_empty_data_flows: true + # Whether to hard delete entities or soft delete them + hard_delete_entities: false + # Keep the last n dataprocess instances + keep_last_n: 5 + soft_deleted_entities_cleanup: + # Delete soft deleted entities which were deleted 10 days ago + retention_days: 10 diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py index c21361eb256c1..cc2a1dc7a7322 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py @@ -23,6 +23,7 @@ ContainerClass, DomainsClass, EmbedClass, + FabricTypeClass, GlobalTagsClass, MetadataChangeEventClass, OwnerClass, @@ -190,6 +191,12 @@ def gen_containers( created: Optional[int] = None, last_modified: Optional[int] = None, ) -> Iterable[MetadataWorkUnit]: + # because of backwards compatibility with a past issue, container_key.env may be a valid env or an instance name + env = ( + container_key.env + if container_key.env in vars(FabricTypeClass).values() + else None + ) container_urn = container_key.as_urn() yield MetadataChangeProposalWrapper( entityUrn=f"{container_urn}", @@ -207,6 +214,7 @@ def gen_containers( lastModified=( TimeStamp(time=last_modified) if last_modified is not None else None ), + env=env if env is not None else None, ), ).as_workunit() diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py index 29228565d0886..489773c5745ff 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py @@ -370,9 +370,9 @@ def _process_schema_worker( yield wu except Exception as e: if self.config.is_profiling_enabled(): - action_mesage = "Does your service account has bigquery.tables.list, bigquery.routines.get, bigquery.routines.list permission, bigquery.tables.getData permission?" + action_mesage = "Does your service account have bigquery.tables.list, bigquery.routines.get, bigquery.routines.list permission, bigquery.tables.getData permission?" else: - action_mesage = "Does your service account has bigquery.tables.list, bigquery.routines.get, bigquery.routines.list permission?" + action_mesage = "Does your service account have bigquery.tables.list, bigquery.routines.get, bigquery.routines.list permission?" self.report.failure( title="Unable to get tables for dataset", diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py index f1fe1f45d75a3..1897f3f288ec0 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py @@ -3,7 +3,8 @@ import re import time from dataclasses import dataclass -from typing import Dict, Iterable +from functools import partial +from typing import Dict, Iterable, List, Optional from pydantic import Field @@ -15,13 +16,29 @@ platform_name, support_status, ) -from datahub.ingestion.api.source import Source, SourceReport +from datahub.ingestion.api.source import MetadataWorkUnitProcessor, Source, SourceReport +from datahub.ingestion.api.source_helpers import auto_workunit_reporter from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.gc.dataprocess_cleanup import ( + DataProcessCleanup, + DataProcessCleanupConfig, + DataProcessCleanupReport, +) +from datahub.ingestion.source.gc.soft_deleted_entity_cleanup import ( + SoftDeletedEntitiesCleanup, + SoftDeletedEntitiesCleanupConfig, + SoftDeletedEntitiesReport, +) logger = logging.getLogger(__name__) class DataHubGcSourceConfig(ConfigModel): + dry_run: bool = Field( + default=False, + description="Whether to perform a dry run or not. This is only supported for dataprocess cleanup and soft deleted entities cleanup.", + ) + cleanup_expired_tokens: bool = Field( default=True, description="Whether to clean up expired tokens or not", @@ -43,9 +60,19 @@ class DataHubGcSourceConfig(ConfigModel): description="Sleep between truncation monitoring.", ) + dataprocess_cleanup: Optional[DataProcessCleanupConfig] = Field( + default=None, + description="Configuration for data process cleanup", + ) + + soft_deleted_entities_cleanup: Optional[SoftDeletedEntitiesCleanupConfig] = Field( + default=None, + description="Configuration for soft deleted entities cleanup", + ) + @dataclass -class DataHubGcSourceReport(SourceReport): +class DataHubGcSourceReport(DataProcessCleanupReport, SoftDeletedEntitiesReport): expired_tokens_revoked: int = 0 @@ -53,17 +80,45 @@ class DataHubGcSourceReport(SourceReport): @config_class(DataHubGcSourceConfig) @support_status(SupportStatus.TESTING) class DataHubGcSource(Source): + """ + DataHubGcSource is responsible for performing garbage collection tasks on DataHub. + + This source performs the following tasks: + 1. Cleans up expired tokens. + 2. Truncates Elasticsearch indices based on configuration. + 3. Cleans up data processes and soft-deleted entities if configured. + + """ + def __init__(self, ctx: PipelineContext, config: DataHubGcSourceConfig): self.ctx = ctx self.config = config self.report = DataHubGcSourceReport() self.graph = ctx.require_graph("The DataHubGc source") + self.dataprocess_cleanup: Optional[DataProcessCleanup] = None + self.soft_deleted_entities_cleanup: Optional[SoftDeletedEntitiesCleanup] = None + + if self.config.dataprocess_cleanup: + self.dataprocess_cleanup = DataProcessCleanup( + ctx, self.config.dataprocess_cleanup, self.report, self.config.dry_run + ) + if self.config.soft_deleted_entities_cleanup: + self.soft_deleted_entities_cleanup = SoftDeletedEntitiesCleanup( + ctx, + self.config.soft_deleted_entities_cleanup, + self.report, + self.config.dry_run, + ) @classmethod def create(cls, config_dict, ctx): config = DataHubGcSourceConfig.parse_obj(config_dict) return cls(ctx, config) + # auto_work_unit_report is overriden to disable a couple of automation like auto status aspect, etc. which is not needed her. + def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: + return [partial(auto_workunit_reporter, self.get_report())] + def get_workunits_internal( self, ) -> Iterable[MetadataWorkUnit]: @@ -71,6 +126,10 @@ def get_workunits_internal( self.revoke_expired_tokens() if self.config.truncate_indices: self.truncate_indices() + if self.dataprocess_cleanup: + yield from self.dataprocess_cleanup.get_workunits_internal() + if self.soft_deleted_entities_cleanup: + self.soft_deleted_entities_cleanup.cleanup_soft_deleted_entities() yield from [] def truncate_indices(self) -> None: diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py new file mode 100644 index 0000000000000..80f7b7a9f4480 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py @@ -0,0 +1,436 @@ +import logging +import time +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from datetime import datetime, timezone +from functools import partial +from typing import Dict, Iterable, List, Optional + +from pydantic import Field + +from datahub.configuration import ConfigModel +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceReport +from datahub.ingestion.api.source_helpers import auto_workunit_reporter +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.graph.client import DataHubGraph +from datahub.utilities.lossy_collections import LossyList +from datahub.utilities.stats_collections import TopKDict + +logger = logging.getLogger(__name__) + +DATAJOB_QUERY = """ +query listDataJobs($query:String!, $scrollId: String, $batchSize: Int) { + scrollAcrossEntities(input: { types: [DATA_JOB + ], query: $query, count: $batchSize, scrollId: $scrollId + }) { + nextScrollId + count + searchResults { + entity { + type + ... on DataJob { + urn + dataFlow { + urn + } + type + lastIngested + subTypes { + typeNames + } + jobId + dataPlatformInstance { + urn + } + runs { + total + } + } + } + } + } +} +""" + +DATAFLOW_QUERY = """ +query listDataFlows($query:String!, $scrollId: String, $batchSize: Int) { + scrollAcrossEntities(input: { types: [DATA_FLOW], query: $query, count: $batchSize, scrollId: $scrollId}) { + nextScrollId + count + searchResults { + entity { + type + ... on DataFlow { + urn + type + orchestrator + cluster + lastIngested + } + } + } + } +} +""" + + +DATA_PROCESS_INSTANCES_QUERY = """ +query getDataJobRuns($dataJobUrn: String!, $start: Int!, $count: Int!) { + dataJob(urn: $dataJobUrn) { + runs(start: $start, count: $count) { + runs { + created { + time + actor + } + urn + } + #...runResult + + __typename + } + __typename + } +} +""" + + +class DataProcessCleanupConfig(ConfigModel): + retention_days: Optional[int] = Field( + 10, + description="Number of days to retain metadata in DataHub", + ) + + aspects_to_clean: List[str] = Field( + ["DataprocessInstance"], + description="List of aspect names to clean up", + ) + + keep_last_n: Optional[int] = Field( + 5, + description="Number of latest aspects to keep", + ) + + delete_empty_data_jobs: bool = Field( + True, description="Wether to delete Data Jobs without runs" + ) + + delete_empty_data_flows: bool = Field( + True, description="Wether to delete Data Flows without runs" + ) + + hard_delete_entities: bool = Field( + False, + description="Whether to hard delete entities", + ) + + batch_size: int = Field( + 500, + description="The number of entities to get in a batch from GraphQL", + ) + + max_workers: int = Field( + 10, + description="The number of workers to use for deletion", + ) + + delay: Optional[float] = Field( + 0.25, + description="Delay between each batch", + ) + + +@dataclass +class DataFlowEntity: + urn: str + orchestrator: Optional[str] + cluster: Optional[str] + last_ingested: Optional[int] + + +@dataclass +class DataJobEntity: + urn: str + flow_urn: str + lastIngested: Optional[int] + jobId: Optional[str] + dataPlatformInstance: Optional[str] + total_runs: int = 0 + + +@dataclass +class DataProcessCleanupReport(SourceReport): + num_aspects_removed: int = 0 + num_aspect_removed_by_type: TopKDict[str, int] = field(default_factory=TopKDict) + sample_removed_aspects_by_type: TopKDict[str, LossyList[str]] = field( + default_factory=TopKDict + ) + + +class DataProcessCleanup: + """ + This source is a maintenance source which cleans up old/unused aspects. + + Currently it only supports:. + - DataFlow + -DataJob + - DataProcessInstance + + """ + + def __init__( + self, + ctx: PipelineContext, + config: DataProcessCleanupConfig, + report: DataProcessCleanupReport, + dry_run: bool = False, + ): + if not ctx.graph: + raise ValueError("MetadataCleanupSource needs a datahub_api") + + self.graph: DataHubGraph = ctx.graph + self.ctx = ctx + self.config = config + self.report = report + self.dry_run = dry_run + + def get_report(self) -> DataProcessCleanupReport: + return self.report + + # auto_work_unit_report is overriden to disable a couple of automation like auto status aspect, etc.. which is not needed her. + def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: + return [partial(auto_workunit_reporter, self.get_report())] + + def fetch_dpis(self, job_urn: str, batch_size: int) -> List[dict]: + assert self.ctx.graph + dpis = [] + start = 0 + while True: + job_query_result = self.ctx.graph.execute_graphql( + DATA_PROCESS_INSTANCES_QUERY, + {"dataJobUrn": job_urn, "start": start, "count": batch_size}, + ) + job_data = job_query_result.get("dataJob") + if not job_data: + raise ValueError(f"Error getting job {job_urn}") + + runs_data = job_data.get("runs") + if not runs_data: + raise ValueError(f"Error getting runs for {job_urn}") + + runs = runs_data.get("runs") + dpis.extend(runs) + start += batch_size + if len(runs) < batch_size: + break + return dpis + + def keep_last_n_dpi( + self, dpis: List[Dict], job: DataJobEntity, executor: ThreadPoolExecutor + ) -> None: + if not self.config.keep_last_n: + return + + deleted_count_last_n = 0 + if len(dpis) >= self.config.keep_last_n: + futures = {} + for dpi in dpis[self.config.keep_last_n :]: + future = executor.submit( + self.delete_entity, dpi["urn"], "dataprocessInstance" + ) + futures[future] = dpi + + for future in as_completed(futures): + deleted_count_last_n += 1 + futures[future]["deleted"] = True + + if deleted_count_last_n % self.config.batch_size == 0: + logger.info(f"Deleted {deleted_count_last_n} DPIs from {job.urn}") + if self.config.delay: + logger.info(f"Sleeping for {self.config.delay} seconds") + time.sleep(self.config.delay) + + logger.info(f"Deleted {deleted_count_last_n} DPIs from {job.urn}") + + def delete_entity(self, urn: str, type: str) -> None: + assert self.ctx.graph + + self.report.num_aspects_removed += 1 + self.report.num_aspect_removed_by_type[type] = ( + self.report.num_aspect_removed_by_type.get(type, 0) + 1 + ) + if type not in self.report.sample_removed_aspects_by_type: + self.report.sample_removed_aspects_by_type[type] = LossyList() + self.report.sample_removed_aspects_by_type[type].append(urn) + + if self.dry_run: + logger.info( + f"Dry run is on otherwise it would have deleted {urn} with hard deletion is{self.config.hard_delete_entities}" + ) + return + + self.ctx.graph.delete_entity(urn, self.config.hard_delete_entities) + + def delete_dpi_from_datajobs(self, job: DataJobEntity) -> None: + assert self.ctx.graph + + dpis = self.fetch_dpis(job.urn, self.config.batch_size) + dpis.sort(key=lambda x: x["created"]["time"], reverse=True) + + with ThreadPoolExecutor(max_workers=self.config.max_workers) as executor: + if self.config.keep_last_n: + self.keep_last_n_dpi(dpis, job, executor) + + if self.config.retention_days is not None: + self.remove_old_dpis(dpis, job, executor) + + job.total_runs = len( + list( + filter(lambda dpi: "deleted" not in dpi or not dpi.get("deleted"), dpis) + ) + ) + + def remove_old_dpis( + self, dpis: List[Dict], job: DataJobEntity, executor: ThreadPoolExecutor + ) -> None: + if self.config.retention_days is None: + return + + deleted_count_retention = 0 + retention_time = ( + int(datetime.now(timezone.utc).timestamp()) + - self.config.retention_days * 24 * 60 * 60 + ) + + futures = {} + for dpi in dpis: + if dpi.get("deleted"): + continue + + if dpi["created"]["time"] < retention_time * 1000: + future = executor.submit( + self.delete_entity, dpi["urn"], "dataprocessInstance" + ) + futures[future] = dpi + + for future in as_completed(futures): + deleted_count_retention += 1 + futures[future]["deleted"] = True + + if deleted_count_retention % self.config.batch_size == 0: + logger.info( + f"Deleted {deleted_count_retention} DPIs from {job.urn} due to retention" + ) + + if self.config.delay: + logger.info(f"Sleeping for {self.config.delay} seconds") + time.sleep(self.config.delay) + + logger.info( + f"Deleted {deleted_count_retention} DPIs from {job.urn} due to retention" + ) + + def get_data_flows(self) -> Iterable[DataFlowEntity]: + assert self.ctx.graph + + scroll_id: Optional[str] = None + previous_scroll_id: Optional[str] = None + + while True: + result = self.ctx.graph.execute_graphql( + DATAFLOW_QUERY, + { + "query": "*", + "scrollId": scroll_id if scroll_id else None, + "batchSize": self.config.batch_size, + }, + ) + scrollAcrossEntities = result.get("scrollAcrossEntities") + if not scrollAcrossEntities: + raise ValueError("Missing scrollAcrossEntities in response") + + scroll_id = scrollAcrossEntities.get("nextScrollId") + for flow in scrollAcrossEntities.get("searchResults"): + yield DataFlowEntity( + urn=flow.get("entity").get("urn"), + orchestrator=flow.get("entity").get("orchestrator"), + cluster=flow.get("entity").get("cluster"), + last_ingested=flow.get("entity").get("lastIngested"), + ) + + if not scroll_id or previous_scroll_id == scroll_id: + break + + previous_scroll_id = scroll_id + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + assert self.ctx.graph + + dataFlows: Dict[str, DataFlowEntity] = {} + for flow in self.get_data_flows(): + dataFlows[flow.urn] = flow + + scroll_id: Optional[str] = None + dataJobs: Dict[str, List[DataJobEntity]] = defaultdict(list) + deleted_jobs: int = 0 + while True: + result = self.ctx.graph.execute_graphql( + DATAJOB_QUERY, + { + "query": "*", + "scrollId": scroll_id if scroll_id else None, + "batchSize": self.config.batch_size, + }, + ) + scrollAcrossEntities = result.get("scrollAcrossEntities") + if not scrollAcrossEntities: + raise ValueError("Missing scrollAcrossEntities in response") + + logger.info(f"Got {scrollAcrossEntities.get('count')} DataJob entities") + + scroll_id = scrollAcrossEntities.get("nextScrollId") + for job in scrollAcrossEntities.get("searchResults"): + datajob_entity = DataJobEntity( + urn=job.get("entity").get("urn"), + flow_urn=job.get("entity").get("dataFlow").get("urn"), + lastIngested=job.get("entity").get("lastIngested"), + jobId=job.get("entity").get("jobId"), + dataPlatformInstance=job.get("entity").get("dataPlatformInstance"), + total_runs=job.get("entity").get("runs").get("total"), + ) + if datajob_entity.total_runs > 0: + self.delete_dpi_from_datajobs(datajob_entity) + if ( + datajob_entity.total_runs == 0 + and self.config.delete_empty_data_jobs + ): + logger.info( + f"Deleting datajob {datajob_entity.urn} because there are no runs" + ) + self.delete_entity(datajob_entity.urn, "dataJob") + deleted_jobs += 1 + if deleted_jobs % self.config.batch_size == 0: + logger.info(f"Deleted {deleted_jobs} DataJobs") + else: + dataJobs[datajob_entity.flow_urn].append(datajob_entity) + + if not scroll_id: + break + + logger.info(f"Deleted {deleted_jobs} DataJobs") + # Delete empty dataflows if needed + if self.config.delete_empty_data_flows: + deleted_data_flows: int = 0 + for key in dataFlows.keys(): + if not dataJobs.get(key) or len(dataJobs[key]) == 0: + logger.info( + f"Deleting dataflow {key} because there are not datajobs" + ) + self.delete_entity(key, "dataFlow") + deleted_data_flows += 1 + if deleted_jobs % self.config.batch_size == 0: + logger.info(f"Deleted {deleted_data_flows} DataFlows") + logger.info(f"Deleted {deleted_data_flows} DataFlows") + return [] diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py new file mode 100644 index 0000000000000..4da23c13659a7 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py @@ -0,0 +1,187 @@ +import logging +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import List, Optional + +from pydantic import Field + +from datahub.configuration import ConfigModel +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.source import SourceReport +from datahub.ingestion.graph.client import DataHubGraph +from datahub.ingestion.graph.filters import RemovedStatusFilter +from datahub.utilities.lossy_collections import LossyList +from datahub.utilities.stats_collections import TopKDict +from datahub.utilities.urns._urn_base import Urn + +logger = logging.getLogger(__name__) + + +class SoftDeletedEntitiesCleanupConfig(ConfigModel): + retention_days: Optional[int] = Field( + 10, + description="Number of days to retain metadata in DataHub", + ) + + batch_size: int = Field( + 500, + description="The number of entities to get in a batch from GraphQL", + ) + + delay: Optional[float] = Field( + 0.25, + description="Delay between each batch", + ) + + max_workers: int = Field( + 10, + description="The number of workers to use for deletion", + ) + + entity_types: Optional[List[str]] = Field( + default=None, + description="List of entity types to cleanup", + ) + + platform: Optional[str] = Field( + default=None, + description="Platform to cleanup", + ) + + env: Optional[str] = Field( + default=None, + description="Environment to cleanup", + ) + + query: Optional[str] = Field( + default=None, + description="Query to filter entities", + ) + + +@dataclass +class SoftDeletedEntitiesReport(SourceReport): + num_soft_deleted_entity_removed: int = 0 + num_soft_deleted_entity_removed_by_type: TopKDict[str, int] = field( + default_factory=TopKDict + ) + sample_soft_deleted_removed_aspects_by_type: TopKDict[str, LossyList[str]] = field( + default_factory=TopKDict + ) + + +class SoftDeletedEntitiesCleanup: + """ + Maintenance source to cleanup soft deleted entities in DataHub + """ + + def __init__( + self, + ctx: PipelineContext, + config: SoftDeletedEntitiesCleanupConfig, + report: SoftDeletedEntitiesReport, + dry_run: bool = False, + ): + if not ctx.graph: + raise ValueError(" Datahub API is required") + + self.graph: DataHubGraph = ctx.graph + self.ctx = ctx + self.config = config + self.report = report + self.dry_run = dry_run + + def delete_entity(self, urn: str) -> None: + assert self.ctx.graph + + entity_urn = Urn.create_from_string(urn) + self.report.num_soft_deleted_entity_removed += 1 + self.report.num_soft_deleted_entity_removed_by_type[entity_urn.entity_type] = ( + self.report.num_soft_deleted_entity_removed_by_type.get( + entity_urn.entity_type, 0 + ) + + 1 + ) + if ( + entity_urn.entity_type + not in self.report.sample_soft_deleted_removed_aspects_by_type + ): + self.report.sample_soft_deleted_removed_aspects_by_type[ + entity_urn.entity_type + ] = LossyList() + self.report.sample_soft_deleted_removed_aspects_by_type[ + entity_urn.entity_type + ].append(urn) + + if self.dry_run: + logger.info( + f"Dry run is on otherwise it would have deleted {urn} with hard deletion" + ) + return + + self.ctx.graph.delete_entity(urn=urn, hard=True) + + def delete_soft_deleted_entity(self, urn: str) -> None: + assert self.ctx.graph + + if self.config.retention_days is None: + logger.info("Retention days is not set, skipping soft delete cleanup") + return + + retention_time = ( + int(datetime.now(timezone.utc).timestamp()) + - self.config.retention_days * 24 * 60 * 60 + ) + + aspect = self.ctx.graph.get_entity_raw(entity_urn=urn, aspects=["status"]) + if "status" in aspect["aspects"]: + if aspect["aspects"]["status"]["value"]["removed"] and aspect["aspects"][ + "status" + ]["created"]["time"] < (retention_time * 1000): + logger.debug(f"Hard deleting {urn}") + self.delete_entity(urn) + + def cleanup_soft_deleted_entities(self) -> None: + assert self.ctx.graph + + deleted_count_retention = 0 + urns = self.ctx.graph.get_urns_by_filter( + entity_types=self.config.entity_types, + platform=self.config.platform, + env=self.config.env, + query=self.config.query, + status=RemovedStatusFilter.ONLY_SOFT_DELETED, + batch_size=self.config.batch_size, + ) + + futures = {} + with ThreadPoolExecutor(max_workers=self.config.max_workers) as executor: + for urn in urns: + future = executor.submit(self.delete_soft_deleted_entity, urn) + futures[future] = urn + + if not futures: + return + for future in as_completed(futures): + if future.exception(): + logger.error( + f"Failed to delete entity {futures[future]}: {future.exception()}" + ) + self.report.failure( + f"Failed to delete entity {futures[future]}", + exc=future.exception(), + ) + deleted_count_retention += 1 + + if deleted_count_retention % self.config.batch_size == 0: + logger.info( + f"Processed {deleted_count_retention} soft deleted entity and deleted {self.report.num_soft_deleted_entity_removed} entities so far" + ) + + if self.config.delay: + logger.debug( + f"Sleeping for {self.config.delay} seconds before getting next batch" + ) + time.sleep(self.config.delay) diff --git a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json index bc9d734218903..537eeb5623149 100644 --- a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json +++ b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "project_id": "project-id-1" }, - "name": "project-id-1" + "name": "project-id-1", + "env": "PROD" } }, "systemMetadata": { @@ -100,7 +101,8 @@ "dataset_id": "bigquery-dataset-1" }, "externalUrl": "https://console.cloud.google.com/bigquery?project=project-id-1&ws=!1m4!1m3!3m2!1sproject-id-1!2sbigquery-dataset-1", - "name": "bigquery-dataset-1" + "name": "bigquery-dataset-1", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_project_label_mcp_golden.json b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_project_label_mcp_golden.json index a529ddc6221a7..ab59b95a9f388 100644 --- a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_project_label_mcp_golden.json +++ b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_project_label_mcp_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "project_id": "dev" }, - "name": "dev" + "name": "dev", + "env": "PROD" } }, "systemMetadata": { @@ -100,7 +101,8 @@ "dataset_id": "bigquery-dataset-1" }, "externalUrl": "https://console.cloud.google.com/bigquery?project=dev&ws=!1m4!1m3!3m2!1sdev!2sbigquery-dataset-1", - "name": "bigquery-dataset-1" + "name": "bigquery-dataset-1", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json b/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json index 3a132d02fdcfb..e35a3dbbf044f 100644 --- a/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json +++ b/metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json @@ -12,12 +12,14 @@ "env": "PROD", "database": "db1" }, - "name": "db1" + "name": "db1", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -32,7 +34,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -48,7 +51,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -65,7 +69,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -85,7 +90,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -100,7 +106,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -124,7 +131,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -146,11 +154,11 @@ "primary_key": "col_Int64", "sampling_key": "", "storage_policy": "default", - "metadata_modification_time": "2023-07-25 07:22:35", + "metadata_modification_time": "2024-08-20 14:33:09", "total_rows": "10", "total_bytes": "671", - "data_paths": "['/var/lib/clickhouse/store/b18/b18ad53b-7256-4a06-8af0-252f2dfcd99d/']", - "metadata_path": "/var/lib/clickhouse/store/1f3/1f37b9b4-f5c5-40f6-8a1b-54c6e2ad5d59/mv_target_table.sql" + "data_paths": "['/var/lib/clickhouse/store/8c1/8c16d9dc-9307-47c4-ba53-6f39f6823f25/']", + "metadata_path": "/var/lib/clickhouse/store/377/37707827-6df2-4c53-b8f6-26b7346f8416/mv_target_table.sql" }, "name": "mv_target_table", "description": "This is target table for materialized view", @@ -250,7 +258,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -266,7 +275,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -283,7 +293,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -307,7 +318,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -322,7 +334,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -344,11 +357,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "default", - "metadata_modification_time": "2023-07-25 07:22:35", + "metadata_modification_time": "2024-08-20 14:33:09", "total_rows": "0", "total_bytes": "0", - "data_paths": "['/var/lib/clickhouse/store/54d/54dc8931-17de-4afd-bc11-636bbcadcdeb/']", - "metadata_path": "/var/lib/clickhouse/store/1f3/1f37b9b4-f5c5-40f6-8a1b-54c6e2ad5d59/test_data_types.sql" + "data_paths": "['/var/lib/clickhouse/store/23a/23a3e7b9-bfb5-4f65-bba2-adc249615387/']", + "metadata_path": "/var/lib/clickhouse/store/377/37707827-6df2-4c53-b8f6-26b7346f8416/test_data_types.sql" }, "name": "test_data_types", "description": "This table has basic types", @@ -877,7 +890,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -893,7 +907,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -910,7 +925,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -934,7 +950,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -949,7 +966,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -973,7 +991,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -995,11 +1014,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", - "metadata_modification_time": "2023-07-25 07:22:35", + "metadata_modification_time": "2024-08-20 14:33:09", "total_rows": "None", "total_bytes": "None", "data_paths": "[]", - "metadata_path": "/var/lib/clickhouse/store/1f3/1f37b9b4-f5c5-40f6-8a1b-54c6e2ad5d59/test_dict.sql" + "metadata_path": "/var/lib/clickhouse/store/377/37707827-6df2-4c53-b8f6-26b7346f8416/test_dict.sql" }, "name": "test_dict", "description": "", @@ -1060,7 +1079,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1076,7 +1096,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1093,7 +1114,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1117,7 +1139,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1132,7 +1155,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1154,11 +1178,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "default", - "metadata_modification_time": "2023-07-25 07:22:35", + "metadata_modification_time": "2024-08-20 14:33:09", "total_rows": "0", "total_bytes": "0", - "data_paths": "['/var/lib/clickhouse/store/320/320b9e3e-757c-4b98-8576-43be9678c69a/']", - "metadata_path": "/var/lib/clickhouse/store/1f3/1f37b9b4-f5c5-40f6-8a1b-54c6e2ad5d59/test_nested_data_types.sql" + "data_paths": "['/var/lib/clickhouse/store/c7b/c7ba2e57-f123-4e22-ac9d-803ddb00de77/']", + "metadata_path": "/var/lib/clickhouse/store/377/37707827-6df2-4c53-b8f6-26b7346f8416/test_nested_data_types.sql" }, "name": "test_nested_data_types", "description": "This table has nested types", @@ -1323,7 +1347,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1339,7 +1364,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1356,7 +1382,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1380,7 +1407,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1395,7 +1423,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1419,7 +1448,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1441,11 +1471,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", - "metadata_modification_time": "2023-07-25 07:22:35", + "metadata_modification_time": "2024-08-20 14:33:09", "total_rows": "None", "total_bytes": "None", - "data_paths": "['/var/lib/clickhouse/store/b18/b18ad53b-7256-4a06-8af0-252f2dfcd99d/']", - "metadata_path": "/var/lib/clickhouse/store/1f3/1f37b9b4-f5c5-40f6-8a1b-54c6e2ad5d59/mv_with_target_table.sql", + "data_paths": "['/var/lib/clickhouse/store/8c1/8c16d9dc-9307-47c4-ba53-6f39f6823f25/']", + "metadata_path": "/var/lib/clickhouse/store/377/37707827-6df2-4c53-b8f6-26b7346f8416/mv_with_target_table.sql", "view_definition": "", "is_view": "True" }, @@ -1547,7 +1577,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1563,7 +1594,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1580,7 +1612,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1597,7 +1630,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1621,7 +1655,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1636,7 +1671,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1660,7 +1696,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1682,11 +1719,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", - "metadata_modification_time": "2023-07-25 07:22:35", + "metadata_modification_time": "2024-08-20 14:33:09", "total_rows": "0", "total_bytes": "0", - "data_paths": "['/var/lib/clickhouse/store/d51/d5151daf-778d-4171-85b6-d9c409ed6ec1/']", - "metadata_path": "/var/lib/clickhouse/store/1f3/1f37b9b4-f5c5-40f6-8a1b-54c6e2ad5d59/mv_without_target_table.sql", + "data_paths": "['/var/lib/clickhouse/store/c17/c17e750d-fa54-4e86-aeec-bfd392b07d71/']", + "metadata_path": "/var/lib/clickhouse/store/377/37707827-6df2-4c53-b8f6-26b7346f8416/mv_without_target_table.sql", "view_definition": "", "is_view": "True" }, @@ -1788,7 +1825,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1804,7 +1842,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1821,7 +1860,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1838,7 +1878,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1862,7 +1903,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1877,7 +1919,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1901,7 +1944,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1923,11 +1967,11 @@ "primary_key": "", "sampling_key": "", "storage_policy": "", - "metadata_modification_time": "2023-07-25 07:22:35", + "metadata_modification_time": "2024-08-20 14:33:09", "total_rows": "None", "total_bytes": "None", "data_paths": "[]", - "metadata_path": "/var/lib/clickhouse/store/1f3/1f37b9b4-f5c5-40f6-8a1b-54c6e2ad5d59/test_view.sql", + "metadata_path": "/var/lib/clickhouse/store/377/37707827-6df2-4c53-b8f6-26b7346f8416/test_view.sql", "view_definition": "", "is_view": "True" }, @@ -1977,7 +2021,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -1993,7 +2038,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2010,7 +2056,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2027,7 +2074,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } }, { @@ -2051,7 +2099,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "clickhouse-test" + "runId": "clickhouse-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json b/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json index 9c731fe9a1ba5..f7ee62201a863 100644 --- a/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json +++ b/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json @@ -146,7 +146,8 @@ "env": "DEV", "folder_abs_path": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "DEV" } }, "systemMetadata": { @@ -233,7 +234,8 @@ "env": "DEV", "folder_abs_path": "my-test-bucket/delta_tables" }, - "name": "delta_tables" + "name": "delta_tables", + "env": "DEV" } }, "systemMetadata": { @@ -354,8 +356,8 @@ "json": { "timestampMillis": 1672531200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "CONVERT", @@ -374,68 +376,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-test-bucket/delta_tables/sales,DEV)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd", - "urn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd" - }, - { - "id": "urn:li:container:acebf8bcf966274632d3d2b710ef4947", - "urn": "urn:li:container:acebf8bcf966274632d3d2b710ef4947" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1672531200000, - "runId": "delta-lake-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1672531200000, - "runId": "delta-lake-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:acebf8bcf966274632d3d2b710ef4947", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd", - "urn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1672531200000, - "runId": "delta-lake-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-test-bucket/delta_tables/sales,DEV)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json index e71ec17250854..24344d6a26664 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json @@ -45,7 +45,7 @@ }, "fields": [ { - "fieldPath": "[version=2.0].[type=int].foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, "type": { "type": { @@ -58,7 +58,7 @@ "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, "type": { "type": { @@ -108,7 +108,8 @@ "env": "UAT", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "UAT" } }, "systemMetadata": { @@ -202,7 +203,8 @@ "env": "UAT", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "UAT" } }, "systemMetadata": { @@ -316,7 +318,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake" }, - "name": "delta_lake" + "name": "delta_lake", + "env": "UAT" } }, "systemMetadata": { @@ -434,7 +437,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "UAT" } }, "systemMetadata": { @@ -556,7 +560,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake/test_data/delta_tables" }, - "name": "delta_tables" + "name": "delta_tables", + "env": "UAT" } }, "systemMetadata": { @@ -694,8 +699,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -724,8 +729,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -754,8 +759,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -784,8 +789,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -814,8 +819,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -921,20 +926,20 @@ }, "fields": [ { - "fieldPath": "[version=2.0].[type=int].year", + "fieldPath": "[version=2.0].[type=float].total_cost", "nullable": true, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "float", "recursive": false, "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" + "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].month", + "fieldPath": "[version=2.0].[type=int].day", "nullable": true, "type": { "type": { @@ -947,7 +952,7 @@ "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].day", + "fieldPath": "[version=2.0].[type=int].month", "nullable": true, "type": { "type": { @@ -960,17 +965,17 @@ "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=string].sale_id", + "fieldPath": "[version=2.0].[type=int].year", "nullable": true, "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { "fieldPath": "[version=2.0].[type=string].customer", @@ -986,17 +991,17 @@ "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=float].total_cost", + "fieldPath": "[version=2.0].[type=string].sale_id", "nullable": true, "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "float", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -1035,8 +1040,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "CONVERT", @@ -1141,7 +1146,7 @@ }, "fields": [ { - "fieldPath": "[version=2.0].[type=int].foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, "type": { "type": { @@ -1154,7 +1159,7 @@ "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, "type": { "type": { @@ -1216,8 +1221,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1246,8 +1251,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1276,8 +1281,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1306,8 +1311,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1336,8 +1341,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1443,7 +1448,7 @@ }, "fields": [ { - "fieldPath": "[version=2.0].[type=int].foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, "type": { "type": { @@ -1456,7 +1461,7 @@ "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, "type": { "type": { @@ -1506,7 +1511,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake/test_data/delta_tables/level1" }, - "name": "level1" + "name": "level1", + "env": "UAT" } }, "systemMetadata": { @@ -1648,8 +1654,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1678,8 +1684,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1708,8 +1714,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1738,8 +1744,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1768,8 +1774,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1788,360 +1794,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" - }, - { - "id": "urn:li:container:189046201d696e7810132cfa64dad337", - "urn": "urn:li:container:189046201d696e7810132cfa64dad337" - }, - { - "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", - "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" - }, - { - "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", - "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" - }, - { - "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", - "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" - }, - { - "id": "urn:li:container:a282913be26fceff334523c2be119df1", - "urn": "urn:li:container:a282913be26fceff334523c2be119df1" - }, - { - "id": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef", - "urn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "allow_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "allow_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" - }, - { - "id": "urn:li:container:189046201d696e7810132cfa64dad337", - "urn": "urn:li:container:189046201d696e7810132cfa64dad337" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "allow_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" - }, - { - "id": "urn:li:container:189046201d696e7810132cfa64dad337", - "urn": "urn:li:container:189046201d696e7810132cfa64dad337" - }, - { - "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", - "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "allow_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" - }, - { - "id": "urn:li:container:189046201d696e7810132cfa64dad337", - "urn": "urn:li:container:189046201d696e7810132cfa64dad337" - }, - { - "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", - "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" - }, - { - "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", - "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "allow_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:a282913be26fceff334523c2be119df1", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" - }, - { - "id": "urn:li:container:189046201d696e7810132cfa64dad337", - "urn": "urn:li:container:189046201d696e7810132cfa64dad337" - }, - { - "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", - "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" - }, - { - "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", - "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" - }, - { - "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", - "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "allow_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" - }, - { - "id": "urn:li:container:189046201d696e7810132cfa64dad337", - "urn": "urn:li:container:189046201d696e7810132cfa64dad337" - }, - { - "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", - "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" - }, - { - "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", - "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" - }, - { - "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", - "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" - }, - { - "id": "urn:li:container:a282913be26fceff334523c2be119df1", - "urn": "urn:li:container:a282913be26fceff334523c2be119df1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "allow_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" - }, - { - "id": "urn:li:container:189046201d696e7810132cfa64dad337", - "urn": "urn:li:container:189046201d696e7810132cfa64dad337" - }, - { - "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", - "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" - }, - { - "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", - "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" - }, - { - "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", - "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" - }, - { - "id": "urn:li:container:a282913be26fceff334523c2be119df1", - "urn": "urn:li:container:a282913be26fceff334523c2be119df1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "allow_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" - }, - { - "id": "urn:li:container:189046201d696e7810132cfa64dad337", - "urn": "urn:li:container:189046201d696e7810132cfa64dad337" - }, - { - "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", - "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" - }, - { - "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", - "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" - }, - { - "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", - "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" - }, - { - "id": "urn:li:container:a282913be26fceff334523c2be119df1", - "urn": "urn:li:container:a282913be26fceff334523c2be119df1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "allow_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" - }, - { - "id": "urn:li:container:189046201d696e7810132cfa64dad337", - "urn": "urn:li:container:189046201d696e7810132cfa64dad337" - }, - { - "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", - "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" - }, - { - "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", - "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" - }, - { - "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", - "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" - }, - { - "id": "urn:li:container:a282913be26fceff334523c2be119df1", - "urn": "urn:li:container:a282913be26fceff334523c2be119df1" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "allow_table.json", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json index 11d6924296f77..717481b253429 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json @@ -45,7 +45,7 @@ }, "fields": [ { - "fieldPath": "[version=2.0].[type=int].foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, "type": { "type": { @@ -58,7 +58,7 @@ "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, "type": { "type": { @@ -107,7 +107,8 @@ "env": "UAT", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "UAT" } }, "systemMetadata": { @@ -194,7 +195,8 @@ "env": "UAT", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "UAT" } }, "systemMetadata": { @@ -302,7 +304,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake" }, - "name": "delta_lake" + "name": "delta_lake", + "env": "UAT" } }, "systemMetadata": { @@ -414,7 +417,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "UAT" } }, "systemMetadata": { @@ -530,7 +534,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake/test_data/delta_tables" }, - "name": "delta_tables" + "name": "delta_tables", + "env": "UAT" } }, "systemMetadata": { @@ -663,8 +668,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -693,8 +698,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -723,8 +728,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -753,8 +758,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -783,8 +788,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -886,20 +891,20 @@ }, "fields": [ { - "fieldPath": "[version=2.0].[type=int].year", + "fieldPath": "[version=2.0].[type=float].total_cost", "nullable": true, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "float", "recursive": false, "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" + "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].month", + "fieldPath": "[version=2.0].[type=int].day", "nullable": true, "type": { "type": { @@ -912,7 +917,7 @@ "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].day", + "fieldPath": "[version=2.0].[type=int].month", "nullable": true, "type": { "type": { @@ -925,17 +930,17 @@ "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=string].sale_id", + "fieldPath": "[version=2.0].[type=int].year", "nullable": true, "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { "fieldPath": "[version=2.0].[type=string].customer", @@ -951,17 +956,17 @@ "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=float].total_cost", + "fieldPath": "[version=2.0].[type=string].sale_id", "nullable": true, "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "float", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -1000,8 +1005,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "CONVERT", @@ -1102,7 +1107,7 @@ }, "fields": [ { - "fieldPath": "[version=2.0].[type=int].foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, "type": { "type": { @@ -1115,7 +1120,7 @@ "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, "type": { "type": { @@ -1177,8 +1182,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1207,8 +1212,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1237,8 +1242,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1267,8 +1272,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1297,8 +1302,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1400,7 +1405,7 @@ }, "fields": [ { - "fieldPath": "[version=2.0].[type=int].foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, "type": { "type": { @@ -1413,7 +1418,7 @@ "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, "type": { "type": { @@ -1462,7 +1467,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake/test_data/delta_tables/level1" }, - "name": "level1" + "name": "level1", + "env": "UAT" } }, "systemMetadata": { @@ -1599,8 +1605,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1629,8 +1635,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1659,8 +1665,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1689,8 +1695,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1719,8 +1725,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1739,319 +1745,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - }, - { - "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" - }, - { - "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", - "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" - }, - { - "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", - "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" - }, - { - "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", - "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" - }, - { - "id": "urn:li:container:6bb6dc6de93177210067d00b45b481bb", - "urn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "inner_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "inner_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "inner_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - }, - { - "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "inner_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - }, - { - "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" - }, - { - "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", - "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "inner_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - }, - { - "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" - }, - { - "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", - "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" - }, - { - "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", - "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "inner_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - }, - { - "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" - }, - { - "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", - "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" - }, - { - "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", - "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" - }, - { - "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", - "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "inner_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - }, - { - "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" - }, - { - "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", - "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" - }, - { - "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", - "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" - }, - { - "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", - "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "inner_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - }, - { - "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" - }, - { - "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", - "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" - }, - { - "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", - "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" - }, - { - "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", - "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "inner_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - }, - { - "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" - }, - { - "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", - "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" - }, - { - "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", - "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" - }, - { - "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", - "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "inner_table.json", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json index ec1912a7e2c49..f446c2deb6a84 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json @@ -45,7 +45,7 @@ }, "fields": [ { - "fieldPath": "[version=2.0].[type=int].foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, "type": { "type": { @@ -58,7 +58,7 @@ "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, "type": { "type": { @@ -107,7 +107,8 @@ "env": "UAT", "folder_abs_path": "delta_tables" }, - "name": "delta_tables" + "name": "delta_tables", + "env": "UAT" } }, "systemMetadata": { @@ -207,8 +208,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -237,8 +238,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -267,8 +268,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -297,8 +298,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -327,8 +328,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -347,43 +348,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:85267d161e1a2ffa647cec6c1188549f", - "urn": "urn:li:container:85267d161e1a2ffa647cec6c1188549f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "relative_path.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:85267d161e1a2ffa647cec6c1188549f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "relative_path.json", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json index ffb4040d17bff..100f93fdaf5d3 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json @@ -44,7 +44,7 @@ }, "fields": [ { - "fieldPath": "[version=2.0].[type=int].foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, "type": { "type": { @@ -57,7 +57,7 @@ "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "[version=2.0].[type=int].bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, "type": { "type": { @@ -106,7 +106,8 @@ "env": "UAT", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "UAT" } }, "systemMetadata": { @@ -193,7 +194,8 @@ "env": "UAT", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "UAT" } }, "systemMetadata": { @@ -301,7 +303,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake" }, - "name": "delta_lake" + "name": "delta_lake", + "env": "UAT" } }, "systemMetadata": { @@ -413,7 +416,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "UAT" } }, "systemMetadata": { @@ -529,7 +533,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake/test_data/delta_tables" }, - "name": "delta_tables" + "name": "delta_tables", + "env": "UAT" } }, "systemMetadata": { @@ -662,8 +667,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -683,167 +688,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - }, - { - "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" - }, - { - "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", - "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" - }, - { - "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", - "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" - }, - { - "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", - "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "single_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "single_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "single_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - }, - { - "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "single_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - }, - { - "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" - }, - { - "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", - "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "single_table.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", - "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" - }, - { - "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", - "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" - }, - { - "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", - "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" - }, - { - "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", - "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "single_table.json", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_tables_with_nested_datatypes.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_tables_with_nested_datatypes.json index fbf4578ef6589..cb40e152f67cc 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_tables_with_nested_datatypes.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_tables_with_nested_datatypes.json @@ -146,7 +146,8 @@ "env": "UAT", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "UAT" } }, "systemMetadata": { @@ -240,7 +241,8 @@ "env": "UAT", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "UAT" } }, "systemMetadata": { @@ -354,7 +356,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake" }, - "name": "delta_lake" + "name": "delta_lake", + "env": "UAT" } }, "systemMetadata": { @@ -472,7 +475,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "UAT" } }, "systemMetadata": { @@ -594,7 +598,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" }, - "name": "delta_tables_nested_datatype" + "name": "delta_tables_nested_datatype", + "env": "UAT" } }, "systemMetadata": { @@ -732,8 +737,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -958,8 +963,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1136,8 +1141,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1340,8 +1345,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1592,8 +1597,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", @@ -1770,8 +1775,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "CUSTOM", "customOperationType": "WRITE", diff --git a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_1.json b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_1.json index 2fad0643e5027..3ba795a5d044a 100644 --- a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_1.json +++ b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_1.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "metastore" }, - "name": "metastore" + "name": "metastore", + "env": "PROD" } }, "systemMetadata": { @@ -99,7 +100,8 @@ "database": "metastore", "schema": "db1" }, - "name": "db1" + "name": "db1", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_2.json b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_2.json index 58e1e11c8dd76..21d6babfc2c97 100644 --- a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_2.json +++ b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_2.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "metastore" }, - "name": "metastore" + "name": "metastore", + "env": "PROD" } }, "systemMetadata": { @@ -99,7 +100,8 @@ "database": "metastore", "schema": "db1" }, - "name": "db1" + "name": "db1", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_3.json b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_3.json index 78db506868679..a9bf2cb26da49 100644 --- a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_3.json +++ b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_3.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "metastore" }, - "name": "metastore" + "name": "metastore", + "env": "PROD" } }, "systemMetadata": { @@ -99,7 +100,8 @@ "database": "metastore", "schema": "db1" }, - "name": "db1" + "name": "db1", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_4.json b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_4.json index 193e1e23b9de4..c764d572f651b 100644 --- a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_4.json +++ b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_4.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "metastore" }, - "name": "metastore" + "name": "metastore", + "env": "PROD" } }, "systemMetadata": { @@ -99,7 +100,8 @@ "database": "metastore", "schema": "db1" }, - "name": "db1" + "name": "db1", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_5.json b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_5.json index ce7ebdd299579..1937550e1bcbd 100644 --- a/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_5.json +++ b/metadata-ingestion/tests/integration/hive-metastore/hive_metastore_mces_golden_5.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "metastore" }, - "name": "metastore" + "name": "metastore", + "env": "PROD" } }, "systemMetadata": { @@ -99,7 +100,8 @@ "database": "metastore", "schema": "db1" }, - "name": "db1" + "name": "db1", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json b/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json index 6774d4c7055b9..b3922f76d7b0c 100644 --- a/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json +++ b/metadata-ingestion/tests/integration/hive/hive_mces_all_db_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "db1" }, - "name": "db1" + "name": "db1", + "env": "PROD" } }, "systemMetadata": { @@ -117,7 +118,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:23 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", @@ -127,7 +128,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1697721972", + "Table Parameters: transient_lastDdlTime": "1724166683", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -267,7 +268,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:23 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", @@ -279,7 +280,7 @@ "Table Parameters: numRows": "1", "Table Parameters: rawDataSize": "32", "Table Parameters: totalSize": "33", - "Table Parameters: transient_lastDdlTime": "1697721976", + "Table Parameters: transient_lastDdlTime": "1724166687", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -457,11 +458,11 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Table Type:": "VIRTUAL_VIEW", - "Table Parameters: transient_lastDdlTime": "1697721978", + "Table Parameters: transient_lastDdlTime": "1724166690", "SerDe Library:": "null", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -638,7 +639,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:31 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", @@ -648,7 +649,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1697721978", + "Table Parameters: transient_lastDdlTime": "1724166691", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -792,7 +793,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", @@ -802,7 +803,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1697721978", + "Table Parameters: transient_lastDdlTime": "1724166690", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -995,7 +996,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:08 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:20 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", @@ -1005,7 +1006,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "5812", - "Table Parameters: transient_lastDdlTime": "1697721968", + "Table Parameters: transient_lastDdlTime": "1724166680", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1157,7 +1158,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:23 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", @@ -1167,7 +1168,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1697721972", + "Table Parameters: transient_lastDdlTime": "1724166683", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1338,14 +1339,14 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test_view_materialized", "Table Type:": "MATERIALIZED_VIEW", "Table Parameters: numFiles": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1697721978", + "Table Parameters: transient_lastDdlTime": "1724166690", "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde", "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat", @@ -1518,7 +1519,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", @@ -1528,7 +1529,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1697721978", + "Table Parameters: transient_lastDdlTime": "1724166690", "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde", "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat", @@ -1767,7 +1768,8 @@ "env": "PROD", "database": "db2" }, - "name": "db2" + "name": "db2", + "env": "PROD" } }, "systemMetadata": { @@ -1873,7 +1875,7 @@ "customProperties": { "Database:": "db2", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:10 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:22 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db2.db/pokes", @@ -1882,7 +1884,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "5812", - "Table Parameters: transient_lastDdlTime": "1697721971", + "Table Parameters: transient_lastDdlTime": "1724166683", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -2003,7 +2005,8 @@ "env": "PROD", "database": "default" }, - "name": "default" + "name": "default", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json index e93924049f626..4a0a4886d606a 100644 --- a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "db1" }, - "name": "db1" + "name": "db1", + "env": "PROD" } }, "systemMetadata": { @@ -117,7 +118,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:23 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", @@ -127,7 +128,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1697721972", + "Table Parameters: transient_lastDdlTime": "1724166683", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -267,7 +268,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:23 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", @@ -279,7 +280,7 @@ "Table Parameters: numRows": "1", "Table Parameters: rawDataSize": "32", "Table Parameters: totalSize": "33", - "Table Parameters: transient_lastDdlTime": "1697721976", + "Table Parameters: transient_lastDdlTime": "1724166687", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -457,11 +458,11 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Table Type:": "VIRTUAL_VIEW", - "Table Parameters: transient_lastDdlTime": "1697721978", + "Table Parameters: transient_lastDdlTime": "1724166690", "SerDe Library:": "null", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -638,7 +639,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:31 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", @@ -648,7 +649,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1697721978", + "Table Parameters: transient_lastDdlTime": "1724166691", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -792,7 +793,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", @@ -802,7 +803,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1697721978", + "Table Parameters: transient_lastDdlTime": "1724166690", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -995,7 +996,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:08 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:20 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", @@ -1005,7 +1006,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "5812", - "Table Parameters: transient_lastDdlTime": "1697721968", + "Table Parameters: transient_lastDdlTime": "1724166680", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1157,7 +1158,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:12 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:23 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", @@ -1167,7 +1168,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1697721972", + "Table Parameters: transient_lastDdlTime": "1724166683", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1338,14 +1339,14 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test_view_materialized", "Table Type:": "MATERIALIZED_VIEW", "Table Parameters: numFiles": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1697721978", + "Table Parameters: transient_lastDdlTime": "1724166690", "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde", "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat", @@ -1518,7 +1519,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Oct 19 13:26:18 UTC 2023", + "CreateTime:": "Tue Aug 20 15:11:30 UTC 2024", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", @@ -1528,7 +1529,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1697721978", + "Table Parameters: transient_lastDdlTime": "1724166690", "SerDe Library:": "org.apache.hadoop.hive.ql.io.orc.OrcSerde", "InputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat", diff --git a/metadata-ingestion/tests/integration/looker/golden_looker_mces.json b/metadata-ingestion/tests/integration/looker/golden_looker_mces.json index 1ce1b4b4750da..5cac7b1bb73b1 100644 --- a/metadata-ingestion/tests/integration/looker/golden_looker_mces.json +++ b/metadata-ingestion/tests/integration/looker/golden_looker_mces.json @@ -238,7 +238,8 @@ "env": "PROD", "model_name": "bogus data" }, - "name": "bogus data" + "name": "bogus data", + "env": "PROD" } }, "systemMetadata": { @@ -334,7 +335,8 @@ "env": "PROD", "model_name": "data" }, - "name": "data" + "name": "data", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json index 87430ef1067ff..24a738a815cda 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json @@ -182,7 +182,8 @@ "env": "PROD", "model_name": "data" }, - "name": "data" + "name": "data", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json b/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json index d893a9bcc50b7..b1460779da4f5 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json @@ -11,7 +11,8 @@ "env": "PROD", "folder_id": "shared-folder-id" }, - "name": "Shared" + "name": "Shared", + "env": "PROD" } }, "systemMetadata": { @@ -419,7 +420,8 @@ "env": "PROD", "model_name": "data" }, - "name": "data" + "name": "data", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json index 411005fcb6f05..74400b9b5cc56 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json @@ -11,7 +11,8 @@ "env": "PROD", "folder_id": "shared-folder-id" }, - "name": "Shared" + "name": "Shared", + "env": "PROD" } }, "systemMetadata": { @@ -419,7 +420,8 @@ "env": "PROD", "model_name": "data" }, - "name": "data" + "name": "data", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/looker/golden_test_folder_path_pattern_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_folder_path_pattern_ingest.json index 5bee9c4adec8d..89241fb52fb63 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_folder_path_pattern_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_folder_path_pattern_ingest.json @@ -11,7 +11,8 @@ "env": "PROD", "folder_id": "a" }, - "name": "A" + "name": "A", + "env": "PROD" } }, "systemMetadata": { @@ -102,7 +103,8 @@ "env": "PROD", "folder_id": "b" }, - "name": "B" + "name": "B", + "env": "PROD" } }, "systemMetadata": { @@ -193,7 +195,8 @@ "env": "PROD", "folder_id": "c" }, - "name": "C" + "name": "C", + "env": "PROD" } }, "systemMetadata": { @@ -510,7 +513,8 @@ "env": "PROD", "model_name": "data" }, - "name": "data" + "name": "data", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json index 6000332b2bf99..63ffdda8c5b6f 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json @@ -11,7 +11,8 @@ "env": "PROD", "folder_id": "shared-folder-id" }, - "name": "Shared" + "name": "Shared", + "env": "PROD" } }, "systemMetadata": { @@ -580,7 +581,8 @@ "env": "PROD", "model_name": "data" }, - "name": "data" + "name": "data", + "env": "PROD" } }, "systemMetadata": { @@ -676,7 +678,8 @@ "env": "PROD", "model_name": "sales_model" }, - "name": "sales_model" + "name": "sales_model", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json index 9c0363e0892f0..d969ef62a96e5 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest.json @@ -12,7 +12,8 @@ "env": "PROD", "folder_id": "shared-folder-id" }, - "name": "Shared" + "name": "Shared", + "env": "PROD" } }, "systemMetadata": { @@ -468,7 +469,8 @@ "env": "PROD", "model_name": "data" }, - "name": "data" + "name": "data", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json index 487a18022cb08..153db363c7828 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json @@ -11,7 +11,8 @@ "env": "PROD", "folder_id": "shared-folder-id" }, - "name": "Shared" + "name": "Shared", + "env": "PROD" } }, "systemMetadata": { @@ -419,7 +420,8 @@ "env": "PROD", "model_name": "data" }, - "name": "data" + "name": "data", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json index 4d8e2e79eafad..98adbdc5b829e 100644 --- a/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json +++ b/metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json @@ -182,7 +182,8 @@ "env": "PROD", "model_name": "data" }, - "name": "data" + "name": "data", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json index ec5a4f1f80159..567ab78a14754 100644 --- a/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json +++ b/metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json @@ -11,7 +11,8 @@ "env": "PROD", "folder_id": "shared-folder-id" }, - "name": "Shared" + "name": "Shared", + "env": "PROD" } }, "systemMetadata": { @@ -433,7 +434,8 @@ "env": "PROD", "model_name": "data" }, - "name": "data" + "name": "data", + "env": "PROD" } }, "systemMetadata": { @@ -808,8 +810,8 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7", + "entityType": "chart", + "entityUrn": "urn:li:chart:(looker,dashboard_elements.10)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -825,8 +827,8 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -859,8 +861,8 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(looker,dashboard_elements.10)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { diff --git a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json index f3da2dbfac2f0..3befb62a631de 100644 --- a/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json +++ b/metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json @@ -134,7 +134,8 @@ "env": "PROD", "model_name": "data" }, - "name": "data" + "name": "data", + "env": "PROD" } }, "systemMetadata": { @@ -453,8 +454,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 25, "favoritesCount": 5, @@ -480,8 +481,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "executionsCount": 14, "uniqueUserCount": 1, @@ -520,8 +521,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "executionsCount": 14, "uniqueUserCount": 1 @@ -546,8 +547,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "executionsCount": 5, "uniqueUserCount": 1, @@ -576,8 +577,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 30 } @@ -601,8 +602,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 10, "userCounts": [ @@ -636,8 +637,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 20 } @@ -661,8 +662,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 35 } diff --git a/metadata-ingestion/tests/integration/lookml/drop_hive_dot_golden.json b/metadata-ingestion/tests/integration/lookml/drop_hive_dot_golden.json index e1dad2e91b735..b78bc973c6574 100644 --- a/metadata-ingestion/tests/integration/lookml/drop_hive_dot_golden.json +++ b/metadata-ingestion/tests/integration/lookml/drop_hive_dot_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/duplicate_field_ingestion_golden.json b/metadata-ingestion/tests/integration/lookml/duplicate_field_ingestion_golden.json index ca9a1503a6854..f88dfba42eb97 100644 --- a/metadata-ingestion/tests/integration/lookml/duplicate_field_ingestion_golden.json +++ b/metadata-ingestion/tests/integration/lookml/duplicate_field_ingestion_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/expected_output.json b/metadata-ingestion/tests/integration/lookml/expected_output.json index f42c600281ccb..7c5605b0edb6d 100644 --- a/metadata-ingestion/tests/integration/lookml/expected_output.json +++ b/metadata-ingestion/tests/integration/lookml/expected_output.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/field_tag_ingestion_golden.json b/metadata-ingestion/tests/integration/lookml/field_tag_ingestion_golden.json index 19352a85249ba..3646f662522ee 100644 --- a/metadata-ingestion/tests/integration/lookml/field_tag_ingestion_golden.json +++ b/metadata-ingestion/tests/integration/lookml/field_tag_ingestion_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json index 5f9b99ebe3062..5253446845f90 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_bigquery.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json index 1b95959f0ba1d..462ab43918351 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_api_hive2.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json index fd479a2baa722..5b39e8dd96ac2 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_badsql_parser.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json index fd479a2baa722..5b39e8dd96ac2 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json index 32386098b7be8..9e2cdb51c08de 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_deny_pattern.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json index 053e90d473c1b..9dff0cb8059de 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_offline_platform_instance.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json b/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json index 44dd72e8fc41b..3ccb71f66f7e3 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_mces_with_external_urls.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json b/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json index 8c3504a736490..9c42b1bdcce0f 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_reachable_views.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/lookml_same_name_views_different_file_path.json b/metadata-ingestion/tests/integration/lookml/lookml_same_name_views_different_file_path.json index b86f7703e6f5e..8fea100adb907 100644 --- a/metadata-ingestion/tests/integration/lookml/lookml_same_name_views_different_file_path.json +++ b/metadata-ingestion/tests/integration/lookml/lookml_same_name_views_different_file_path.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/refinement_include_order_golden.json b/metadata-ingestion/tests/integration/lookml/refinement_include_order_golden.json index c5b1d44772dea..10ead0550e6db 100644 --- a/metadata-ingestion/tests/integration/lookml/refinement_include_order_golden.json +++ b/metadata-ingestion/tests/integration/lookml/refinement_include_order_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_refinement_sample1" }, - "name": "lkml_refinement_sample1" + "name": "lkml_refinement_sample1", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json b/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json index 7c2f92ac1e028..d2d7ad093a6d0 100644 --- a/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json +++ b/metadata-ingestion/tests/integration/lookml/refinements_ingestion_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json index dd6917c112579..468450310c2ab 100644 --- a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json +++ b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "project_name": "lkml_samples" }, - "name": "lkml_samples" + "name": "lkml_samples", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json index bba160984eed8..2666990ed1333 100644 --- a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json +++ b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json @@ -12,7 +12,8 @@ "env": "PROD", "database": "mngdb" }, - "name": "mngdb" + "name": "mngdb", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_small_schema_size_golden.json b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_small_schema_size_golden.json index b2a1ba03dab76..577a053f1c741 100644 --- a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_small_schema_size_golden.json +++ b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_small_schema_size_golden.json @@ -12,7 +12,8 @@ "env": "PROD", "database": "mngdb" }, - "name": "mngdb" + "name": "mngdb", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json index bddccc856c842..14b03619de4c1 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "dataCharmer" }, - "name": "dataCharmer" + "name": "dataCharmer", + "env": "PROD" } }, "systemMetadata": { @@ -434,8 +435,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 10, "columnCount": 6, @@ -592,8 +593,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 112, "columnCount": 4, @@ -1012,7 +1013,8 @@ "env": "PROD", "database": "metagalaxy" }, - "name": "metagalaxy" + "name": "metagalaxy", + "env": "PROD" } }, "systemMetadata": { @@ -1722,7 +1724,8 @@ "env": "PROD", "database": "northwind" }, - "name": "northwind" + "name": "northwind", + "env": "PROD" } }, "systemMetadata": { @@ -2145,8 +2148,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 5, "columnCount": 6, @@ -2277,8 +2280,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 0, "columnCount": 3, @@ -2320,7 +2323,8 @@ "env": "PROD", "database": "test_cases" }, - "name": "test_cases" + "name": "test_cases", + "env": "PROD" } }, "systemMetadata": { @@ -2636,8 +2640,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 0, "columnCount": 1, @@ -2666,8 +2670,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 0, "columnCount": 1, diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json index 8c6f6338bc2b0..a6a627e5bfcfc 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "northwind" }, - "name": "northwind" + "name": "northwind", + "env": "PROD" } }, "systemMetadata": { @@ -477,8 +478,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 5, "columnCount": 6, @@ -609,8 +610,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 0, "columnCount": 3, diff --git a/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json b/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json index 3f5899aa8a98c..71d3dcab5e116 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "northwind" }, - "name": "northwind" + "name": "northwind", + "env": "PROD" } }, "systemMetadata": { @@ -423,8 +424,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 5, "columnCount": 6, @@ -447,8 +448,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 0, "columnCount": 3, diff --git a/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json b/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json index 58a70cae2b2d5..fcf0baa35f4a3 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "northwind" }, - "name": "northwind" + "name": "northwind", + "env": "PROD" } }, "systemMetadata": { @@ -423,8 +424,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 0, "columnCount": 6, @@ -484,8 +485,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 0, "columnCount": 3, diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json index 4054d757c22fd..cbcadde6feb21 100644 --- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json +++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "OraDoc" }, - "name": "OraDoc" + "name": "OraDoc", + "env": "PROD" } }, "systemMetadata": { @@ -99,7 +100,8 @@ "database": "OraDoc", "schema": "schema1" }, - "name": "schema1" + "name": "schema1", + "env": "PROD" } }, "systemMetadata": { @@ -595,7 +597,8 @@ "database": "OraDoc", "schema": "schema2" }, - "name": "schema2" + "name": "schema2", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json index 9df23078be18d..dc0208586d1a1 100644 --- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json +++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "OraDoc" }, - "name": "OraDoc" + "name": "OraDoc", + "env": "PROD" } }, "systemMetadata": { @@ -99,7 +100,8 @@ "database": "OraDoc", "schema": "schema1" }, - "name": "schema1" + "name": "schema1", + "env": "PROD" } }, "systemMetadata": { @@ -595,7 +597,8 @@ "database": "OraDoc", "schema": "schema2" }, - "name": "schema2" + "name": "schema2", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json index f42ff7c0df068..21898ca246b65 100644 --- a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "postgres" }, - "name": "postgres" + "name": "postgres", + "env": "PROD" } }, "systemMetadata": { @@ -99,7 +100,8 @@ "database": "postgres", "schema": "public" }, - "name": "public" + "name": "public", + "env": "PROD" } }, "systemMetadata": { @@ -207,7 +209,8 @@ "env": "PROD", "database": "postgrestest" }, - "name": "postgrestest" + "name": "postgrestest", + "env": "PROD" } }, "systemMetadata": { @@ -295,7 +298,8 @@ "database": "postgrestest", "schema": "public" }, - "name": "public" + "name": "public", + "env": "PROD" } }, "systemMetadata": { @@ -769,8 +773,8 @@ "json": { "timestampMillis": 1646575200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 2, "columnCount": 9, diff --git a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json index f107fb1006bf6..fc4a0affac561 100644 --- a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "postgrestest" }, - "name": "postgrestest" + "name": "postgrestest", + "env": "PROD" } }, "systemMetadata": { @@ -99,7 +100,8 @@ "database": "postgrestest", "schema": "public" }, - "name": "public" + "name": "public", + "env": "PROD" } }, "systemMetadata": { @@ -595,8 +597,8 @@ "json": { "timestampMillis": 1646575200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 2, "columnCount": 9, diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_file_without_extension.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_file_without_extension.json index 63efc79941d82..77db69d197c78 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_file_without_extension.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_file_without_extension.json @@ -12,7 +12,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -34,7 +35,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -127,7 +129,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -139,8 +142,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688445129021 @@ -148,7 +151,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -164,12 +168,14 @@ "env": "DEV", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -184,7 +190,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -200,7 +207,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -217,7 +225,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -237,7 +246,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -253,12 +263,14 @@ "env": "DEV", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -273,7 +285,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -289,7 +302,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -306,7 +320,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -321,7 +336,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -345,7 +361,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -361,12 +378,14 @@ "env": "DEV", "folder_abs_path": "tests/integration/s3" }, - "name": "s3" + "name": "s3", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -381,7 +400,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -397,7 +417,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -414,7 +435,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -429,7 +451,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -457,7 +480,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -473,12 +497,14 @@ "env": "DEV", "folder_abs_path": "tests/integration/s3/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -493,7 +519,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -509,7 +536,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -526,7 +554,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -541,7 +570,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -573,7 +603,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -589,12 +620,14 @@ "env": "DEV", "folder_abs_path": "tests/integration/s3/test_data/local_system" }, - "name": "local_system" + "name": "local_system", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -609,7 +642,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -625,7 +659,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -642,7 +677,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -657,7 +693,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -693,7 +730,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -709,12 +747,14 @@ "env": "DEV", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -729,7 +769,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -745,7 +786,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -762,7 +804,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -777,7 +820,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -817,7 +861,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -833,12 +878,14 @@ "env": "DEV", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -853,7 +900,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -869,7 +917,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -886,7 +935,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -901,7 +951,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -945,7 +996,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -961,12 +1013,14 @@ "env": "DEV", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -981,7 +1035,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -997,7 +1052,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1014,7 +1070,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1029,7 +1086,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1077,7 +1135,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1093,12 +1152,14 @@ "env": "DEV", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/no_extension" }, - "name": "no_extension" + "name": "no_extension", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1113,7 +1174,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1129,7 +1191,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1146,7 +1209,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1161,7 +1225,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1213,7 +1278,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1228,7 +1294,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1243,7 +1310,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1299,7 +1367,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json index 03d2ac38cf7d2..b35fb24d43bf3 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json @@ -138,7 +138,8 @@ "env": "UAT", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "UAT" } }, "systemMetadata": { @@ -225,7 +226,8 @@ "env": "UAT", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "UAT" } }, "systemMetadata": { @@ -333,7 +335,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3" }, - "name": "s3" + "name": "s3", + "env": "UAT" } }, "systemMetadata": { @@ -445,7 +448,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "UAT" } }, "systemMetadata": { @@ -561,7 +565,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system" }, - "name": "local_system" + "name": "local_system", + "env": "UAT" } }, "systemMetadata": { @@ -681,7 +686,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { @@ -805,7 +811,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { @@ -933,7 +940,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json index 153ff7799987b..62ba688990e2b 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json @@ -18,7 +18,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -111,7 +112,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -123,8 +125,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688445119021 @@ -132,7 +134,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -147,12 +150,14 @@ "env": "UAT", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -167,7 +172,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -182,7 +188,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -199,7 +206,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -214,7 +222,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -229,12 +238,14 @@ "env": "UAT", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -249,7 +260,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -264,7 +276,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -281,7 +294,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -296,7 +310,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -316,7 +331,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -331,12 +347,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3" }, - "name": "s3" + "name": "s3", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -351,7 +369,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -366,7 +385,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -383,7 +403,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -398,7 +419,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -422,7 +444,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -437,12 +460,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -457,7 +482,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -472,7 +498,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -489,7 +516,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -504,7 +532,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -532,7 +561,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -547,12 +577,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system" }, - "name": "local_system" + "name": "local_system", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -567,7 +599,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -582,7 +615,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -599,7 +633,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -614,7 +649,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -646,7 +682,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -661,12 +698,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -681,7 +720,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -696,7 +736,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -713,7 +754,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -728,7 +770,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -764,7 +807,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -779,12 +823,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -799,7 +845,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -814,7 +861,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -831,7 +879,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -846,7 +895,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -886,7 +936,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -901,12 +952,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -921,7 +974,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -936,7 +990,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -953,7 +1008,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -968,7 +1024,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1012,7 +1069,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1027,7 +1085,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1039,8 +1098,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 70, "columnCount": 5, @@ -1331,7 +1390,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1346,7 +1406,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1394,7 +1455,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json index 48c59753fcaff..346c2e9bcd83a 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json @@ -138,7 +138,8 @@ "env": "UAT", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "UAT" } }, "systemMetadata": { @@ -225,7 +226,8 @@ "env": "UAT", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "UAT" } }, "systemMetadata": { @@ -333,7 +335,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3" }, - "name": "s3" + "name": "s3", + "env": "UAT" } }, "systemMetadata": { @@ -445,7 +448,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "UAT" } }, "systemMetadata": { @@ -561,7 +565,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system" }, - "name": "local_system" + "name": "local_system", + "env": "UAT" } }, "systemMetadata": { @@ -681,7 +686,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { @@ -805,7 +811,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { @@ -933,7 +940,8 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json index 6878e6f618233..6bff1bf1b1468 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json @@ -18,7 +18,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -111,7 +112,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -123,8 +125,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688445119021 @@ -132,7 +134,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -147,12 +150,14 @@ "env": "UAT", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -167,7 +172,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -182,7 +188,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -199,7 +206,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -214,7 +222,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -229,12 +238,14 @@ "env": "UAT", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -249,7 +260,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -264,7 +276,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -281,7 +294,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -296,7 +310,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -316,7 +331,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -331,12 +347,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3" }, - "name": "s3" + "name": "s3", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -351,7 +369,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -366,7 +385,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -383,7 +403,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -398,7 +419,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -422,7 +444,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -437,12 +460,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -457,7 +482,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -472,7 +498,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -489,7 +516,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -504,7 +532,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -532,7 +561,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -547,12 +577,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system" }, - "name": "local_system" + "name": "local_system", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -567,7 +599,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -582,7 +615,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -599,7 +633,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -614,7 +649,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -646,7 +682,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -661,12 +698,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -681,7 +720,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -696,7 +736,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -713,7 +754,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -728,7 +770,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -764,7 +807,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -779,12 +823,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -799,7 +845,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -814,7 +861,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -831,7 +879,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -846,7 +895,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -886,7 +936,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -901,12 +952,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -921,7 +974,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -936,7 +990,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -953,7 +1008,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -968,7 +1024,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1012,7 +1069,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1027,7 +1085,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1039,8 +1098,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 70, "columnCount": 5, @@ -1331,7 +1390,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1346,7 +1406,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1394,7 +1455,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json index b7025839833e1..6668e4e6a26c0 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json @@ -18,7 +18,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -543,7 +544,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -555,8 +557,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688445259021 @@ -564,7 +566,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -579,12 +582,14 @@ "env": "UAT", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -599,7 +604,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -614,7 +620,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -631,7 +638,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -646,7 +654,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -661,12 +670,14 @@ "env": "UAT", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -681,7 +692,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -696,7 +708,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -713,7 +726,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -728,7 +742,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -748,7 +763,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -763,12 +779,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3" }, - "name": "s3" + "name": "s3", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -783,7 +801,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -798,7 +817,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -815,7 +835,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -830,7 +851,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -854,7 +876,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -869,12 +892,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -889,7 +914,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -904,7 +930,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -921,7 +948,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -936,7 +964,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -964,7 +993,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -979,12 +1009,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system" }, - "name": "local_system" + "name": "local_system", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -999,7 +1031,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1014,7 +1047,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1031,7 +1065,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1046,7 +1081,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1078,7 +1114,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1093,12 +1130,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1113,7 +1152,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1128,7 +1168,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1145,7 +1186,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1160,7 +1202,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1196,7 +1239,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1211,12 +1255,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1231,7 +1277,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1246,7 +1293,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1263,7 +1311,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1278,7 +1327,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1318,7 +1368,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1333,12 +1384,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1353,7 +1406,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1368,7 +1422,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1385,7 +1440,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1400,7 +1456,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1444,7 +1501,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1459,7 +1517,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1471,8 +1530,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 1, "columnCount": 9, @@ -1584,7 +1643,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1599,7 +1659,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1647,7 +1708,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json index e94bf407b9fda..7f8bcfec6d314 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json @@ -18,7 +18,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -543,7 +544,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -555,8 +557,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688445259021 @@ -564,7 +566,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -579,12 +582,14 @@ "env": "UAT", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -599,7 +604,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -614,7 +620,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -631,7 +638,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -646,7 +654,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -661,12 +670,14 @@ "env": "UAT", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -681,7 +692,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -696,7 +708,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -713,7 +726,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -728,7 +742,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -748,7 +763,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -763,12 +779,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3" }, - "name": "s3" + "name": "s3", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -783,7 +801,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -798,7 +817,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -815,7 +835,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -830,7 +851,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -854,7 +876,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -869,12 +892,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -889,7 +914,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -904,7 +930,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -921,7 +948,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -936,7 +964,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -964,7 +993,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -979,12 +1009,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system" }, - "name": "local_system" + "name": "local_system", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -999,7 +1031,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1014,7 +1047,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1031,7 +1065,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1046,7 +1081,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1078,7 +1114,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1093,12 +1130,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1113,7 +1152,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1128,7 +1168,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1145,7 +1186,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1160,7 +1202,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1196,7 +1239,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1211,12 +1255,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1231,7 +1277,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1246,7 +1293,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1263,7 +1311,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1278,7 +1327,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1318,7 +1368,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1333,12 +1384,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1353,7 +1406,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1368,7 +1422,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1385,7 +1440,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1400,7 +1456,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1444,7 +1501,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1459,7 +1517,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1471,8 +1530,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 1, "columnCount": 9, @@ -1584,7 +1643,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1599,7 +1659,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1647,7 +1708,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_update_schema.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_update_schema.json index 3aa5813b74c23..35efe52994837 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_update_schema.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_update_schema.json @@ -18,7 +18,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -543,7 +544,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -555,8 +557,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688445259021 @@ -564,7 +566,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -579,12 +582,14 @@ "env": "UAT", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -599,7 +604,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -614,7 +620,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -631,7 +638,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -646,7 +654,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -661,12 +670,14 @@ "env": "UAT", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -681,7 +692,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -696,7 +708,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -713,7 +726,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -728,7 +742,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -748,7 +763,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -763,12 +779,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3" }, - "name": "s3" + "name": "s3", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -783,7 +801,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -798,7 +817,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -815,7 +835,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -830,7 +851,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -854,7 +876,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -869,12 +892,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -889,7 +914,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -904,7 +930,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -921,7 +948,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -936,7 +964,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -964,7 +993,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -979,12 +1009,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system" }, - "name": "local_system" + "name": "local_system", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -999,7 +1031,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1014,7 +1047,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1031,7 +1065,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1046,7 +1081,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1078,7 +1114,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1093,12 +1130,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1113,7 +1152,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1128,7 +1168,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1145,7 +1186,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1160,7 +1202,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1196,7 +1239,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1211,12 +1255,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1231,7 +1277,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1246,7 +1293,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1263,7 +1311,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1278,7 +1327,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1318,7 +1368,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1333,12 +1384,14 @@ "env": "UAT", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1353,7 +1406,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1368,7 +1422,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1385,7 +1440,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1400,7 +1456,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1444,7 +1501,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1459,7 +1517,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1471,8 +1530,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 1, "columnCount": 9, @@ -1584,7 +1643,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1599,7 +1659,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1647,7 +1708,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json index 3c5b0027ea8ad..4f98d68f8ae62 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json @@ -454,8 +454,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688444889021 @@ -480,7 +480,8 @@ "env": "DEV", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "DEV" } }, "systemMetadata": { @@ -574,7 +575,8 @@ "env": "DEV", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "DEV" } }, "systemMetadata": { @@ -688,7 +690,8 @@ "env": "DEV", "folder_abs_path": "tests/integration/s3" }, - "name": "s3" + "name": "s3", + "env": "DEV" } }, "systemMetadata": { @@ -806,7 +809,8 @@ "env": "DEV", "folder_abs_path": "tests/integration/s3/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "DEV" } }, "systemMetadata": { @@ -928,7 +932,8 @@ "env": "DEV", "folder_abs_path": "tests/integration/s3/test_data/local_system" }, - "name": "local_system" + "name": "local_system", + "env": "DEV" } }, "systemMetadata": { @@ -1054,7 +1059,8 @@ "env": "DEV", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "DEV" } }, "systemMetadata": { @@ -1184,7 +1190,8 @@ "env": "DEV", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "DEV" } }, "systemMetadata": { @@ -1318,7 +1325,8 @@ "env": "DEV", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "DEV" } }, "systemMetadata": { @@ -1468,8 +1476,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 33, "columnCount": 14, @@ -2953,8 +2961,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688444899021 @@ -2991,8 +2999,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 28, "columnCount": 5, @@ -3529,8 +3537,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688444909021 @@ -3567,8 +3575,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 28, "columnCount": 5, @@ -4081,8 +4089,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688444919021 @@ -4119,8 +4127,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 1, "columnCount": 1, @@ -4323,8 +4331,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688444929021 @@ -4378,8 +4386,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 70, "columnCount": 5, @@ -4870,8 +4878,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688444939021 @@ -4908,8 +4916,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 5, "columnCount": 5, @@ -5554,8 +5562,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688444949021 @@ -5592,8 +5600,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 249, "columnCount": 31, diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_spec_for_files.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_spec_for_files.json index ed2c992655a89..491cbdf8b9704 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_spec_for_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_spec_for_files.json @@ -18,7 +18,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -111,7 +112,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -123,8 +125,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688444899021 @@ -132,7 +134,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -147,12 +150,14 @@ "env": "PROD", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -167,7 +172,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -182,7 +188,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -199,7 +206,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -214,7 +222,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -229,12 +238,14 @@ "env": "PROD", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -249,7 +260,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -264,7 +276,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -281,7 +294,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -296,7 +310,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -316,7 +331,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -331,12 +347,14 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3" }, - "name": "s3" + "name": "s3", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -351,7 +369,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -366,7 +385,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -383,7 +403,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -398,7 +419,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -422,7 +444,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -437,12 +460,14 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -457,7 +482,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -472,7 +498,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -489,7 +516,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -504,7 +532,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -532,7 +561,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -547,12 +577,14 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data/local_system" }, - "name": "local_system" + "name": "local_system", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -567,7 +599,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -582,7 +615,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -599,7 +633,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -614,7 +649,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -646,7 +682,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -661,12 +698,14 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -681,7 +720,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -696,7 +736,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -713,7 +754,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -728,7 +770,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -764,7 +807,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -779,12 +823,14 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -799,7 +845,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -814,7 +861,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -831,7 +879,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -846,7 +895,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -886,7 +936,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -901,12 +952,14 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -921,7 +974,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -936,7 +990,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -953,7 +1008,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -968,7 +1024,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1012,7 +1069,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1027,7 +1085,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1039,8 +1098,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 28, "columnCount": 5, @@ -1377,7 +1436,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1425,7 +1485,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1447,7 +1508,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1540,7 +1602,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1552,8 +1615,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688444909021 @@ -1561,7 +1624,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1576,7 +1640,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1588,8 +1653,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 28, "columnCount": 5, @@ -1926,7 +1991,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1974,7 +2040,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1989,7 +2056,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -2004,7 +2072,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_specs_of_different_buckets.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_specs_of_different_buckets.json index f7793140fe033..da7c76876d415 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_specs_of_different_buckets.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_specs_of_different_buckets.json @@ -18,7 +18,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -111,7 +112,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -123,8 +125,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688444899021 @@ -132,7 +134,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -147,12 +150,14 @@ "env": "PROD", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -167,7 +172,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -182,7 +188,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -199,7 +206,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -214,7 +222,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -229,12 +238,14 @@ "env": "PROD", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -249,7 +260,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -264,7 +276,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -281,7 +294,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -296,7 +310,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -316,7 +331,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -331,12 +347,14 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3" }, - "name": "s3" + "name": "s3", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -351,7 +369,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -366,7 +385,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -383,7 +403,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -398,7 +419,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -422,7 +444,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -437,12 +460,14 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -457,7 +482,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -472,7 +498,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -489,7 +516,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -504,7 +532,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -532,7 +561,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -547,12 +577,14 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data/local_system" }, - "name": "local_system" + "name": "local_system", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -567,7 +599,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -582,7 +615,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -599,7 +633,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -614,7 +649,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -646,7 +682,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -661,12 +698,14 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -681,7 +720,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -696,7 +736,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -713,7 +754,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -728,7 +770,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -764,7 +807,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -779,12 +823,14 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -799,7 +845,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -814,7 +861,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -831,7 +879,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -846,7 +895,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -886,7 +936,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -901,12 +952,14 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -921,7 +974,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -936,7 +990,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -953,7 +1008,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -968,7 +1024,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1012,7 +1069,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1027,7 +1085,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1039,8 +1098,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 28, "columnCount": 5, @@ -1377,7 +1436,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1425,7 +1485,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1447,7 +1508,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1540,7 +1602,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1552,8 +1615,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688444909021 @@ -1561,7 +1624,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1576,7 +1640,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1588,8 +1653,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 28, "columnCount": 5, @@ -1926,7 +1991,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1974,7 +2040,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1989,7 +2056,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -2004,7 +2072,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_single_file.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_single_file.json index d9a5b8b4a7eb8..76e562142e39e 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_single_file.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_single_file.json @@ -125,8 +125,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688444899021 @@ -150,7 +150,8 @@ "env": "PROD", "folder_abs_path": "tests" }, - "name": "tests" + "name": "tests", + "env": "PROD" } }, "systemMetadata": { @@ -237,7 +238,8 @@ "env": "PROD", "folder_abs_path": "tests/integration" }, - "name": "integration" + "name": "integration", + "env": "PROD" } }, "systemMetadata": { @@ -345,7 +347,8 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3" }, - "name": "s3" + "name": "s3", + "env": "PROD" } }, "systemMetadata": { @@ -457,7 +460,8 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data" }, - "name": "test_data" + "name": "test_data", + "env": "PROD" } }, "systemMetadata": { @@ -573,7 +577,8 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data/local_system" }, - "name": "local_system" + "name": "local_system", + "env": "PROD" } }, "systemMetadata": { @@ -693,7 +698,8 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "PROD" } }, "systemMetadata": { @@ -817,7 +823,8 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "PROD" } }, "systemMetadata": { @@ -945,7 +952,8 @@ "env": "PROD", "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "PROD" } }, "systemMetadata": { @@ -1090,8 +1098,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 28, "columnCount": 5, diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_inference_without_extension.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_inference_without_extension.json index d50f00efacaa0..abc6eb1b471b2 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_inference_without_extension.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_inference_without_extension.json @@ -12,7 +12,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -34,7 +35,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -127,7 +129,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -139,8 +142,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1586848010000 @@ -148,7 +151,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -164,12 +168,14 @@ "env": "DEV", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_inference_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -184,7 +190,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -200,7 +207,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -217,7 +225,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -237,7 +246,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -253,12 +263,14 @@ "env": "DEV", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_inference_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -273,7 +285,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -289,7 +302,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -306,7 +320,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -321,7 +336,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -345,7 +361,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -361,12 +378,14 @@ "env": "DEV", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_inference_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -381,7 +400,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -397,7 +417,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -414,7 +435,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -429,7 +451,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -457,7 +480,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -473,12 +497,14 @@ "env": "DEV", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_inference_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -493,7 +519,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -509,7 +536,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -526,7 +554,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -541,7 +570,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -573,7 +603,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -589,12 +620,14 @@ "env": "DEV", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa/no_extension" }, - "name": "no_extension" + "name": "no_extension", + "env": "DEV" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_inference_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -609,7 +642,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -625,7 +659,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -642,7 +677,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -657,7 +693,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -693,7 +730,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -708,7 +746,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -723,7 +762,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -763,7 +803,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json index 06678512199bf..1c022fabf9158 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json @@ -146,7 +146,8 @@ "env": "DEV", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "DEV" } }, "systemMetadata": { @@ -240,7 +241,8 @@ "env": "DEV", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "DEV" } }, "systemMetadata": { @@ -376,7 +378,8 @@ "env": "DEV", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "DEV" } }, "systemMetadata": { @@ -448,7 +451,8 @@ "env": "DEV", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "DEV" } }, "systemMetadata": { @@ -616,7 +620,8 @@ "env": "DEV", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa/no_extension" }, - "name": "no_extension" + "name": "no_extension", + "env": "DEV" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json index 8e1daed0070f1..41484bec81935 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json @@ -114,7 +114,8 @@ "env": "UAT", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "UAT" } }, "systemMetadata": { @@ -223,7 +224,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { @@ -276,7 +278,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { @@ -443,7 +446,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json index 0d4c2d14d4ac1..0b28381fce8ff 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json @@ -126,7 +126,8 @@ "env": "UAT", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "UAT" } }, "systemMetadata": { @@ -235,7 +236,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { @@ -288,7 +290,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { @@ -455,7 +458,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json index 7172bba73f01c..c4cfed8bfc7ac 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json @@ -114,7 +114,8 @@ "env": "UAT", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "UAT" } }, "systemMetadata": { @@ -223,7 +224,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { @@ -276,7 +278,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { @@ -443,7 +446,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json index 0372a6afd9457..ae81f60ac8dcc 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json @@ -126,7 +126,8 @@ "env": "UAT", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "UAT" } }, "systemMetadata": { @@ -235,7 +236,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { @@ -288,7 +290,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { @@ -455,7 +458,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json index 6ff3925b1afb3..ca598d06076e3 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json @@ -565,7 +565,8 @@ "env": "UAT", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "UAT" } }, "systemMetadata": { @@ -699,7 +700,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { @@ -770,7 +772,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { @@ -903,7 +906,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json index 86fa7835008dd..0dc1c4016d895 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json @@ -565,7 +565,8 @@ "env": "UAT", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "UAT" } }, "systemMetadata": { @@ -699,7 +700,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { @@ -770,7 +772,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { @@ -903,7 +906,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json index 95ec5e83f7991..756036e8c704d 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json @@ -565,7 +565,8 @@ "env": "UAT", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "UAT" } }, "systemMetadata": { @@ -699,7 +700,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { @@ -770,7 +772,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { @@ -903,7 +906,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema_with_partition_autodetect.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema_with_partition_autodetect.json index 4fee8fb36b13f..684af901e6832 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema_with_partition_autodetect.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema_with_partition_autodetect.json @@ -114,7 +114,8 @@ "env": "UAT", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "UAT" } }, "systemMetadata": { @@ -223,7 +224,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { @@ -276,7 +278,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { @@ -443,7 +446,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_all.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_all.json index 13ba57f5be671..4d23cadbbc4d3 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_all.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_all.json @@ -114,7 +114,8 @@ "env": "UAT", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "UAT" } }, "systemMetadata": { @@ -223,7 +224,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { @@ -276,7 +278,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { @@ -443,7 +446,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_min_max.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_min_max.json index 9f45583f1d3ec..6017a27a88895 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_min_max.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_min_max.json @@ -114,7 +114,8 @@ "env": "UAT", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "UAT" } }, "systemMetadata": { @@ -223,7 +224,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "UAT" } }, "systemMetadata": { @@ -276,7 +278,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "UAT" } }, "systemMetadata": { @@ -443,7 +446,8 @@ "env": "UAT", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "UAT" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json index d97cedd39d77c..90a361219c1bf 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json @@ -551,7 +551,8 @@ "env": "DEV", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "DEV" } }, "systemMetadata": { @@ -799,7 +800,8 @@ "env": "DEV", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "DEV" } }, "systemMetadata": { @@ -853,7 +855,8 @@ "env": "DEV", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "DEV" } }, "systemMetadata": { @@ -1443,7 +1446,8 @@ "env": "DEV", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "DEV" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json index 5d45a2a5f696e..c67977ef7fa1b 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json @@ -128,7 +128,8 @@ "env": "PROD", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "PROD" } }, "systemMetadata": { @@ -244,7 +245,8 @@ "env": "PROD", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "PROD" } }, "systemMetadata": { @@ -265,7 +267,8 @@ "env": "PROD", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "PROD" } }, "systemMetadata": { @@ -480,7 +483,8 @@ "env": "PROD", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json index 90c3e0b3850b7..d96bdce912082 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json @@ -128,7 +128,8 @@ "env": "PROD", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "PROD" } }, "systemMetadata": { @@ -244,7 +245,8 @@ "env": "PROD", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "PROD" } }, "systemMetadata": { @@ -265,7 +267,8 @@ "env": "PROD", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "PROD" } }, "systemMetadata": { @@ -480,7 +483,8 @@ "env": "PROD", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "PROD" } }, "systemMetadata": { @@ -778,7 +782,8 @@ "env": "PROD", "bucket_name": "my-test-bucket-2" }, - "name": "my-test-bucket-2" + "name": "my-test-bucket-2", + "env": "PROD" } }, "systemMetadata": { @@ -894,7 +899,8 @@ "env": "PROD", "folder_abs_path": "my-test-bucket-2/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "PROD" } }, "systemMetadata": { @@ -1002,7 +1008,8 @@ "env": "PROD", "folder_abs_path": "my-test-bucket-2/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "PROD" } }, "systemMetadata": { @@ -1114,7 +1121,8 @@ "env": "PROD", "folder_abs_path": "my-test-bucket-2/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json index 5fb691c1ee457..7703d137ddd29 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json @@ -128,7 +128,8 @@ "env": "PROD", "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aa" + "name": "folder_aa", + "env": "PROD" } }, "systemMetadata": { @@ -244,7 +245,8 @@ "env": "PROD", "bucket_name": "my-test-bucket" }, - "name": "my-test-bucket" + "name": "my-test-bucket", + "env": "PROD" } }, "systemMetadata": { @@ -265,7 +267,8 @@ "env": "PROD", "folder_abs_path": "my-test-bucket/folder_a" }, - "name": "folder_a" + "name": "folder_a", + "env": "PROD" } }, "systemMetadata": { @@ -480,7 +483,8 @@ "env": "PROD", "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aaa" + "name": "folder_aaa", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json index 9247c4cb04e3f..4415b1ad3e515 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json @@ -14,6 +14,7 @@ "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/", "name": "TEST_DB", "description": "Comment for TEST_DB", + "env": "PROD", "created": { "time": 1623103200000 }, @@ -24,7 +25,7 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2024_08_20-16_18_24", "lastRunId": "no-run-id-provided" } }, @@ -147,6 +148,7 @@ "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/", "name": "TEST_SCHEMA", "description": "comment for TEST_DB.TEST_SCHEMA", + "env": "PROD", "created": { "time": 1623103200000 }, @@ -157,7 +159,7 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2024_08_20-16_18_24", "lastRunId": "no-run-id-provided" } }, @@ -3936,8 +3938,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 10000, "columnCount": 10, @@ -3959,8 +3961,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 10000, "columnCount": 10, @@ -3982,8 +3984,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 10000, "columnCount": 10, @@ -4005,8 +4007,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 10000, "columnCount": 10, @@ -4028,8 +4030,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 10000, "columnCount": 10, @@ -4051,8 +4053,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 10000, "columnCount": 10, @@ -4074,8 +4076,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 10000, "columnCount": 10, @@ -4097,8 +4099,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 10000, "columnCount": 10, @@ -4120,8 +4122,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 10000, "columnCount": 10, @@ -4143,8 +4145,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 10000, "columnCount": 10, @@ -6628,8 +6630,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "uniqueUserCount": 0, "totalSqlQueries": 0, @@ -6657,8 +6659,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "uniqueUserCount": 0, "totalSqlQueries": 0, @@ -6702,8 +6704,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "uniqueUserCount": 0, "totalSqlQueries": 0, @@ -6731,8 +6733,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "uniqueUserCount": 0, "totalSqlQueries": 0, @@ -6760,8 +6762,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "uniqueUserCount": 0, "totalSqlQueries": 0, @@ -6833,8 +6835,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "uniqueUserCount": 0, "totalSqlQueries": 0, @@ -6931,8 +6933,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "uniqueUserCount": 0, "totalSqlQueries": 0, @@ -6988,8 +6990,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "uniqueUserCount": 0, "totalSqlQueries": 0, @@ -7225,8 +7227,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "uniqueUserCount": 0, "totalSqlQueries": 0, @@ -7323,8 +7325,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "uniqueUserCount": 0, "totalSqlQueries": 0, @@ -7380,8 +7382,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "uniqueUserCount": 0, "totalSqlQueries": 0, @@ -7573,8 +7575,8 @@ "multiple": 1 }, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "uniqueUserCount": 0, "totalSqlQueries": 0, @@ -7598,8 +7600,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "actor": "urn:li:corpuser:abc@xyz.com", "operationType": "CREATE", @@ -7621,8 +7623,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "actor": "urn:li:corpuser:abc@xyz.com", "operationType": "CREATE", @@ -7644,8 +7646,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "actor": "urn:li:corpuser:abc@xyz.com", "operationType": "CREATE", @@ -7667,8 +7669,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "actor": "urn:li:corpuser:abc@xyz.com", "operationType": "CREATE", @@ -7690,8 +7692,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "actor": "urn:li:corpuser:abc@xyz.com", "operationType": "CREATE", @@ -7713,8 +7715,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "actor": "urn:li:corpuser:abc@xyz.com", "operationType": "CREATE", @@ -7736,8 +7738,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "actor": "urn:li:corpuser:abc@xyz.com", "operationType": "CREATE", @@ -7759,8 +7761,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "actor": "urn:li:corpuser:abc@xyz.com", "operationType": "CREATE", @@ -7782,8 +7784,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "actor": "urn:li:corpuser:abc@xyz.com", "operationType": "CREATE", @@ -7805,8 +7807,8 @@ "json": { "timestampMillis": 1654621200000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "actor": "urn:li:corpuser:abc@xyz.com", "operationType": "CREATE", diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json index 2697b7315acab..d232ae710e891 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json @@ -15,6 +15,7 @@ "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/", "name": "TEST_DB", "description": "Comment for TEST_DB", + "env": "PROD", "created": { "time": 1623110400000 }, @@ -225,6 +226,7 @@ "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/", "name": "TEST_SCHEMA", "description": "comment for TEST_DB.TEST_SCHEMA", + "env": "PROD", "created": { "time": 1623110400000 }, diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json index 1213f78e27486..558548bfc7a69 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "DemoData" }, - "name": "DemoData" + "name": "DemoData", + "env": "PROD" } }, "systemMetadata": { @@ -112,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "a7684ed9-7d4c-46f2-b8bd-1dc5941345d0", + "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-03 11:56:19.400000", - "date_modified": "2024-09-03 11:56:19.580000", + "date_created": "2024-09-16 15:59:53.077000", + "date_modified": "2024-09-16 15:59:53.217000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -166,7 +167,8 @@ "database": "DemoData", "schema": "db_accessadmin" }, - "name": "db_accessadmin" + "name": "db_accessadmin", + "env": "PROD" } }, "systemMetadata": { @@ -275,7 +277,8 @@ "database": "DemoData", "schema": "db_backupoperator" }, - "name": "db_backupoperator" + "name": "db_backupoperator", + "env": "PROD" } }, "systemMetadata": { @@ -384,7 +387,8 @@ "database": "DemoData", "schema": "db_datareader" }, - "name": "db_datareader" + "name": "db_datareader", + "env": "PROD" } }, "systemMetadata": { @@ -493,7 +497,8 @@ "database": "DemoData", "schema": "db_datawriter" }, - "name": "db_datawriter" + "name": "db_datawriter", + "env": "PROD" } }, "systemMetadata": { @@ -602,7 +607,8 @@ "database": "DemoData", "schema": "db_ddladmin" }, - "name": "db_ddladmin" + "name": "db_ddladmin", + "env": "PROD" } }, "systemMetadata": { @@ -711,7 +717,8 @@ "database": "DemoData", "schema": "db_denydatareader" }, - "name": "db_denydatareader" + "name": "db_denydatareader", + "env": "PROD" } }, "systemMetadata": { @@ -820,7 +827,8 @@ "database": "DemoData", "schema": "db_denydatawriter" }, - "name": "db_denydatawriter" + "name": "db_denydatawriter", + "env": "PROD" } }, "systemMetadata": { @@ -929,7 +937,8 @@ "database": "DemoData", "schema": "db_owner" }, - "name": "db_owner" + "name": "db_owner", + "env": "PROD" } }, "systemMetadata": { @@ -1038,7 +1047,8 @@ "database": "DemoData", "schema": "db_securityadmin" }, - "name": "db_securityadmin" + "name": "db_securityadmin", + "env": "PROD" } }, "systemMetadata": { @@ -1147,7 +1157,8 @@ "database": "DemoData", "schema": "dbo" }, - "name": "dbo" + "name": "dbo", + "env": "PROD" } }, "systemMetadata": { @@ -1388,7 +1399,8 @@ "database": "DemoData", "schema": "Foo" }, - "name": "Foo" + "name": "Foo", + "env": "PROD" } }, "systemMetadata": { @@ -1961,8 +1973,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-03 11:56:19.310000", - "date_modified": "2024-09-03 11:56:19.310000" + "date_created": "2024-09-16 15:59:53.010000", + "date_modified": "2024-09-16 15:59:53.010000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2008,7 +2020,8 @@ "database": "DemoData", "schema": "guest" }, - "name": "guest" + "name": "guest", + "env": "PROD" } }, "systemMetadata": { @@ -2117,7 +2130,8 @@ "database": "DemoData", "schema": "INFORMATION_SCHEMA" }, - "name": "INFORMATION_SCHEMA" + "name": "INFORMATION_SCHEMA", + "env": "PROD" } }, "systemMetadata": { @@ -2226,7 +2240,8 @@ "database": "DemoData", "schema": "sys" }, - "name": "sys" + "name": "sys", + "env": "PROD" } }, "systemMetadata": { @@ -2334,7 +2349,8 @@ "env": "PROD", "database": "NewData" }, - "name": "NewData" + "name": "NewData", + "env": "PROD" } }, "systemMetadata": { @@ -2422,7 +2438,8 @@ "database": "NewData", "schema": "db_accessadmin" }, - "name": "db_accessadmin" + "name": "db_accessadmin", + "env": "PROD" } }, "systemMetadata": { @@ -2531,7 +2548,8 @@ "database": "NewData", "schema": "db_backupoperator" }, - "name": "db_backupoperator" + "name": "db_backupoperator", + "env": "PROD" } }, "systemMetadata": { @@ -2640,7 +2658,8 @@ "database": "NewData", "schema": "db_datareader" }, - "name": "db_datareader" + "name": "db_datareader", + "env": "PROD" } }, "systemMetadata": { @@ -2749,7 +2768,8 @@ "database": "NewData", "schema": "db_datawriter" }, - "name": "db_datawriter" + "name": "db_datawriter", + "env": "PROD" } }, "systemMetadata": { @@ -2858,7 +2878,8 @@ "database": "NewData", "schema": "db_ddladmin" }, - "name": "db_ddladmin" + "name": "db_ddladmin", + "env": "PROD" } }, "systemMetadata": { @@ -2967,7 +2988,8 @@ "database": "NewData", "schema": "db_denydatareader" }, - "name": "db_denydatareader" + "name": "db_denydatareader", + "env": "PROD" } }, "systemMetadata": { @@ -3076,7 +3098,8 @@ "database": "NewData", "schema": "db_denydatawriter" }, - "name": "db_denydatawriter" + "name": "db_denydatawriter", + "env": "PROD" } }, "systemMetadata": { @@ -3185,7 +3208,8 @@ "database": "NewData", "schema": "db_owner" }, - "name": "db_owner" + "name": "db_owner", + "env": "PROD" } }, "systemMetadata": { @@ -3294,7 +3318,8 @@ "database": "NewData", "schema": "db_securityadmin" }, - "name": "db_securityadmin" + "name": "db_securityadmin", + "env": "PROD" } }, "systemMetadata": { @@ -3403,7 +3428,8 @@ "database": "NewData", "schema": "dbo" }, - "name": "dbo" + "name": "dbo", + "env": "PROD" } }, "systemMetadata": { @@ -3656,7 +3682,8 @@ "database": "NewData", "schema": "FooNew" }, - "name": "FooNew" + "name": "FooNew", + "env": "PROD" } }, "systemMetadata": { @@ -4065,7 +4092,8 @@ "database": "NewData", "schema": "guest" }, - "name": "guest" + "name": "guest", + "env": "PROD" } }, "systemMetadata": { @@ -4174,7 +4202,8 @@ "database": "NewData", "schema": "INFORMATION_SCHEMA" }, - "name": "INFORMATION_SCHEMA" + "name": "INFORMATION_SCHEMA", + "env": "PROD" } }, "systemMetadata": { @@ -4283,7 +4312,8 @@ "database": "NewData", "schema": "sys" }, - "name": "sys" + "name": "sys", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index 7c854fb1bdde8..161d40ea91d91 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "DemoData" }, - "name": "DemoData" + "name": "DemoData", + "env": "PROD" } }, "systemMetadata": { @@ -112,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "a7684ed9-7d4c-46f2-b8bd-1dc5941345d0", + "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-03 11:56:19.400000", - "date_modified": "2024-09-03 11:56:19.580000", + "date_created": "2024-09-16 15:59:53.077000", + "date_modified": "2024-09-16 15:59:53.217000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -166,7 +167,8 @@ "database": "DemoData", "schema": "db_accessadmin" }, - "name": "db_accessadmin" + "name": "db_accessadmin", + "env": "PROD" } }, "systemMetadata": { @@ -275,7 +277,8 @@ "database": "DemoData", "schema": "db_backupoperator" }, - "name": "db_backupoperator" + "name": "db_backupoperator", + "env": "PROD" } }, "systemMetadata": { @@ -384,7 +387,8 @@ "database": "DemoData", "schema": "db_datareader" }, - "name": "db_datareader" + "name": "db_datareader", + "env": "PROD" } }, "systemMetadata": { @@ -493,7 +497,8 @@ "database": "DemoData", "schema": "db_datawriter" }, - "name": "db_datawriter" + "name": "db_datawriter", + "env": "PROD" } }, "systemMetadata": { @@ -602,7 +607,8 @@ "database": "DemoData", "schema": "db_ddladmin" }, - "name": "db_ddladmin" + "name": "db_ddladmin", + "env": "PROD" } }, "systemMetadata": { @@ -711,7 +717,8 @@ "database": "DemoData", "schema": "db_denydatareader" }, - "name": "db_denydatareader" + "name": "db_denydatareader", + "env": "PROD" } }, "systemMetadata": { @@ -820,7 +827,8 @@ "database": "DemoData", "schema": "db_denydatawriter" }, - "name": "db_denydatawriter" + "name": "db_denydatawriter", + "env": "PROD" } }, "systemMetadata": { @@ -929,7 +937,8 @@ "database": "DemoData", "schema": "db_owner" }, - "name": "db_owner" + "name": "db_owner", + "env": "PROD" } }, "systemMetadata": { @@ -1038,7 +1047,8 @@ "database": "DemoData", "schema": "db_securityadmin" }, - "name": "db_securityadmin" + "name": "db_securityadmin", + "env": "PROD" } }, "systemMetadata": { @@ -1147,7 +1157,8 @@ "database": "DemoData", "schema": "dbo" }, - "name": "dbo" + "name": "dbo", + "env": "PROD" } }, "systemMetadata": { @@ -1388,7 +1399,8 @@ "database": "DemoData", "schema": "Foo" }, - "name": "Foo" + "name": "Foo", + "env": "PROD" } }, "systemMetadata": { @@ -1961,8 +1973,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-03 11:56:19.310000", - "date_modified": "2024-09-03 11:56:19.310000" + "date_created": "2024-09-16 15:59:53.010000", + "date_modified": "2024-09-16 15:59:53.010000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2008,7 +2020,8 @@ "database": "DemoData", "schema": "guest" }, - "name": "guest" + "name": "guest", + "env": "PROD" } }, "systemMetadata": { @@ -2117,7 +2130,8 @@ "database": "DemoData", "schema": "INFORMATION_SCHEMA" }, - "name": "INFORMATION_SCHEMA" + "name": "INFORMATION_SCHEMA", + "env": "PROD" } }, "systemMetadata": { @@ -2226,7 +2240,8 @@ "database": "DemoData", "schema": "sys" }, - "name": "sys" + "name": "sys", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json index 7c854fb1bdde8..161d40ea91d91 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "DemoData" }, - "name": "DemoData" + "name": "DemoData", + "env": "PROD" } }, "systemMetadata": { @@ -112,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "a7684ed9-7d4c-46f2-b8bd-1dc5941345d0", + "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-03 11:56:19.400000", - "date_modified": "2024-09-03 11:56:19.580000", + "date_created": "2024-09-16 15:59:53.077000", + "date_modified": "2024-09-16 15:59:53.217000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -166,7 +167,8 @@ "database": "DemoData", "schema": "db_accessadmin" }, - "name": "db_accessadmin" + "name": "db_accessadmin", + "env": "PROD" } }, "systemMetadata": { @@ -275,7 +277,8 @@ "database": "DemoData", "schema": "db_backupoperator" }, - "name": "db_backupoperator" + "name": "db_backupoperator", + "env": "PROD" } }, "systemMetadata": { @@ -384,7 +387,8 @@ "database": "DemoData", "schema": "db_datareader" }, - "name": "db_datareader" + "name": "db_datareader", + "env": "PROD" } }, "systemMetadata": { @@ -493,7 +497,8 @@ "database": "DemoData", "schema": "db_datawriter" }, - "name": "db_datawriter" + "name": "db_datawriter", + "env": "PROD" } }, "systemMetadata": { @@ -602,7 +607,8 @@ "database": "DemoData", "schema": "db_ddladmin" }, - "name": "db_ddladmin" + "name": "db_ddladmin", + "env": "PROD" } }, "systemMetadata": { @@ -711,7 +717,8 @@ "database": "DemoData", "schema": "db_denydatareader" }, - "name": "db_denydatareader" + "name": "db_denydatareader", + "env": "PROD" } }, "systemMetadata": { @@ -820,7 +827,8 @@ "database": "DemoData", "schema": "db_denydatawriter" }, - "name": "db_denydatawriter" + "name": "db_denydatawriter", + "env": "PROD" } }, "systemMetadata": { @@ -929,7 +937,8 @@ "database": "DemoData", "schema": "db_owner" }, - "name": "db_owner" + "name": "db_owner", + "env": "PROD" } }, "systemMetadata": { @@ -1038,7 +1047,8 @@ "database": "DemoData", "schema": "db_securityadmin" }, - "name": "db_securityadmin" + "name": "db_securityadmin", + "env": "PROD" } }, "systemMetadata": { @@ -1147,7 +1157,8 @@ "database": "DemoData", "schema": "dbo" }, - "name": "dbo" + "name": "dbo", + "env": "PROD" } }, "systemMetadata": { @@ -1388,7 +1399,8 @@ "database": "DemoData", "schema": "Foo" }, - "name": "Foo" + "name": "Foo", + "env": "PROD" } }, "systemMetadata": { @@ -1961,8 +1973,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-03 11:56:19.310000", - "date_modified": "2024-09-03 11:56:19.310000" + "date_created": "2024-09-16 15:59:53.010000", + "date_modified": "2024-09-16 15:59:53.010000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2008,7 +2020,8 @@ "database": "DemoData", "schema": "guest" }, - "name": "guest" + "name": "guest", + "env": "PROD" } }, "systemMetadata": { @@ -2117,7 +2130,8 @@ "database": "DemoData", "schema": "INFORMATION_SCHEMA" }, - "name": "INFORMATION_SCHEMA" + "name": "INFORMATION_SCHEMA", + "env": "PROD" } }, "systemMetadata": { @@ -2226,7 +2240,8 @@ "database": "DemoData", "schema": "sys" }, - "name": "sys" + "name": "sys", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json index 1d971624939f8..29124f6fc156c 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "DemoData" }, - "name": "DemoData" + "name": "DemoData", + "env": "PROD" } }, "systemMetadata": { @@ -112,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "a7684ed9-7d4c-46f2-b8bd-1dc5941345d0", + "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-03 11:56:19.400000", - "date_modified": "2024-09-03 11:56:19.580000", + "date_created": "2024-09-16 15:59:53.077000", + "date_modified": "2024-09-16 15:59:53.217000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -166,7 +167,8 @@ "database": "DemoData", "schema": "db_accessadmin" }, - "name": "db_accessadmin" + "name": "db_accessadmin", + "env": "PROD" } }, "systemMetadata": { @@ -275,7 +277,8 @@ "database": "DemoData", "schema": "db_backupoperator" }, - "name": "db_backupoperator" + "name": "db_backupoperator", + "env": "PROD" } }, "systemMetadata": { @@ -384,7 +387,8 @@ "database": "DemoData", "schema": "db_datareader" }, - "name": "db_datareader" + "name": "db_datareader", + "env": "PROD" } }, "systemMetadata": { @@ -493,7 +497,8 @@ "database": "DemoData", "schema": "db_datawriter" }, - "name": "db_datawriter" + "name": "db_datawriter", + "env": "PROD" } }, "systemMetadata": { @@ -602,7 +607,8 @@ "database": "DemoData", "schema": "db_ddladmin" }, - "name": "db_ddladmin" + "name": "db_ddladmin", + "env": "PROD" } }, "systemMetadata": { @@ -711,7 +717,8 @@ "database": "DemoData", "schema": "db_denydatareader" }, - "name": "db_denydatareader" + "name": "db_denydatareader", + "env": "PROD" } }, "systemMetadata": { @@ -820,7 +827,8 @@ "database": "DemoData", "schema": "db_denydatawriter" }, - "name": "db_denydatawriter" + "name": "db_denydatawriter", + "env": "PROD" } }, "systemMetadata": { @@ -929,7 +937,8 @@ "database": "DemoData", "schema": "db_owner" }, - "name": "db_owner" + "name": "db_owner", + "env": "PROD" } }, "systemMetadata": { @@ -1038,7 +1047,8 @@ "database": "DemoData", "schema": "db_securityadmin" }, - "name": "db_securityadmin" + "name": "db_securityadmin", + "env": "PROD" } }, "systemMetadata": { @@ -1147,7 +1157,8 @@ "database": "DemoData", "schema": "dbo" }, - "name": "dbo" + "name": "dbo", + "env": "PROD" } }, "systemMetadata": { @@ -1388,7 +1399,8 @@ "database": "DemoData", "schema": "Foo" }, - "name": "Foo" + "name": "Foo", + "env": "PROD" } }, "systemMetadata": { @@ -1961,8 +1973,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-03 11:56:19.310000", - "date_modified": "2024-09-03 11:56:19.310000" + "date_created": "2024-09-16 15:59:53.010000", + "date_modified": "2024-09-16 15:59:53.010000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2008,7 +2020,8 @@ "database": "DemoData", "schema": "guest" }, - "name": "guest" + "name": "guest", + "env": "PROD" } }, "systemMetadata": { @@ -2117,7 +2130,8 @@ "database": "DemoData", "schema": "INFORMATION_SCHEMA" }, - "name": "INFORMATION_SCHEMA" + "name": "INFORMATION_SCHEMA", + "env": "PROD" } }, "systemMetadata": { @@ -2226,7 +2240,8 @@ "database": "DemoData", "schema": "sys" }, - "name": "sys" + "name": "sys", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json index d19b83f4d4e2f..6745268ea2c24 100644 --- a/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json @@ -12,7 +12,8 @@ "env": "PROD", "database": "hivedb" }, - "name": "hivedb" + "name": "hivedb", + "env": "PROD" } }, "systemMetadata": { @@ -107,7 +108,8 @@ "database": "hivedb", "schema": "db1" }, - "name": "db1" + "name": "db1", + "env": "PROD" } }, "systemMetadata": { @@ -244,7 +246,7 @@ "numrows": "1", "rawdatasize": "32", "totalsize": "33", - "transient_lastddltime": "1722106707" + "transient_lastddltime": "1724180599" }, "name": "array_struct_test", "description": "This table has array of structs", @@ -505,7 +507,7 @@ "numrows": "3", "rawdatasize": "94", "totalsize": "97", - "transient_lastddltime": "1722106711" + "transient_lastddltime": "1724180605" }, "name": "classification_test", "tags": [] @@ -764,7 +766,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1722106709" + "transient_lastddltime": "1724180602" }, "name": "map_test", "tags": [] @@ -991,7 +993,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1722106709" + "transient_lastddltime": "1724180602" }, "name": "nested_struct_test", "tags": [] @@ -1262,7 +1264,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1722106702" + "transient_lastddltime": "1724180591" }, "name": "pokes", "tags": [] @@ -1497,7 +1499,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1722106704" + "transient_lastddltime": "1724180595" }, "name": "struct_test", "tags": [] @@ -1748,7 +1750,7 @@ "customProperties": { "numfiles": "0", "totalsize": "0", - "transient_lastddltime": "1722106709" + "transient_lastddltime": "1724180601" }, "name": "struct_test_view_materialized", "tags": [] @@ -2002,7 +2004,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1722106704" + "transient_lastddltime": "1724180595" }, "name": "_test_table_underscore", "tags": [] @@ -2225,7 +2227,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1722106709" + "transient_lastddltime": "1724180602" }, "name": "union_test", "tags": [] @@ -2527,7 +2529,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1722106709", + "transient_lastddltime": "1724180602", "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"", "is_view": "True" }, diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json index f72610fba7c54..34acf6a6e369b 100644 --- a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "hivedb" }, - "name": "hivedb" + "name": "hivedb", + "env": "PROD" } }, "systemMetadata": { @@ -99,7 +100,8 @@ "database": "hivedb", "schema": "db1" }, - "name": "db1" + "name": "db1", + "env": "PROD" } }, "systemMetadata": { @@ -231,7 +233,7 @@ "numrows": "1", "rawdatasize": "32", "totalsize": "33", - "transient_lastddltime": "1722106707" + "transient_lastddltime": "1724180599" }, "name": "array_struct_test", "description": "This table has array of structs", @@ -471,7 +473,7 @@ "numrows": "3", "rawdatasize": "94", "totalsize": "97", - "transient_lastddltime": "1722106711" + "transient_lastddltime": "1724180605" }, "name": "classification_test", "tags": [] @@ -753,7 +755,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1722106709" + "transient_lastddltime": "1724180602" }, "name": "map_test", "tags": [] @@ -959,7 +961,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1722106709" + "transient_lastddltime": "1724180602" }, "name": "nested_struct_test", "tags": [] @@ -1209,7 +1211,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1722106702" + "transient_lastddltime": "1724180591" }, "name": "pokes", "tags": [] @@ -1423,7 +1425,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1722106704" + "transient_lastddltime": "1724180595" }, "name": "struct_test", "tags": [] @@ -1653,7 +1655,7 @@ "customProperties": { "numfiles": "0", "totalsize": "0", - "transient_lastddltime": "1722106709" + "transient_lastddltime": "1724180601" }, "name": "struct_test_view_materialized", "tags": [] @@ -1886,7 +1888,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1722106704" + "transient_lastddltime": "1724180595" }, "name": "_test_table_underscore", "tags": [] @@ -2088,7 +2090,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1722106709" + "transient_lastddltime": "1724180602" }, "name": "union_test", "tags": [] @@ -2369,7 +2371,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1722106709", + "transient_lastddltime": "1724180602", "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"", "is_view": "True" }, diff --git a/metadata-ingestion/tests/integration/trino/trino_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_mces_golden.json index b2afad81b12fa..a8c40fec5d6c7 100644 --- a/metadata-ingestion/tests/integration/trino/trino_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_mces_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "postgresqldb" }, - "name": "postgresqldb" + "name": "postgresqldb", + "env": "PROD" } }, "systemMetadata": { @@ -99,7 +100,8 @@ "database": "postgresqldb", "schema": "librarydb" }, - "name": "librarydb" + "name": "librarydb", + "env": "PROD" } }, "systemMetadata": { @@ -1109,8 +1111,8 @@ "json": { "timestampMillis": 1632398400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 3, "columnCount": 6, @@ -1194,8 +1196,8 @@ "json": { "timestampMillis": 1632398400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 2, "columnCount": 4, @@ -1281,8 +1283,8 @@ "json": { "timestampMillis": 1632398400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 2, "columnCount": 2, @@ -1332,8 +1334,8 @@ "json": { "timestampMillis": 1632398400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 1, "columnCount": 6, diff --git a/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json b/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json index c6d24371bd6ea..2a32af56d6337 100644 --- a/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json +++ b/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json @@ -12,13 +12,15 @@ "catalog": "hive_metastore" }, "externalUrl": "https://dummy.cloud.databricks.com/explore/data/hive_metastore", - "name": "hive_metastore" + "name": "hive_metastore", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1638860400000, "runId": "unity-catalog-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "unity-catalog-test-pipeline" } }, { @@ -101,13 +103,15 @@ "unity_schema": "bronze_kambi" }, "externalUrl": "https://dummy.cloud.databricks.com/explore/data/hive_metastore/bronze_kambi", - "name": "bronze_kambi" + "name": "bronze_kambi", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1638860400000, "runId": "unity-catalog-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "unity-catalog-test-pipeline" } }, { @@ -1618,13 +1622,15 @@ }, "externalUrl": "https://dummy.cloud.databricks.com/explore/data/quickstart_catalog", "name": "quickstart_catalog", - "description": "" + "description": "", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1638860400000, "runId": "unity-catalog-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "unity-catalog-test-pipeline" } }, { @@ -2316,13 +2322,15 @@ }, "externalUrl": "https://dummy.cloud.databricks.com/explore/data/quickstart_catalog/quickstart_schema", "name": "quickstart_schema", - "description": "A new Unity Catalog schema called quickstart_schema" + "description": "A new Unity Catalog schema called quickstart_schema", + "env": "PROD" } }, "systemMetadata": { "lastObserved": 1638860400000, "runId": "unity-catalog-test", - "lastRunId": "no-run-id-provided" + "lastRunId": "no-run-id-provided", + "pipelineName": "unity-catalog-test-pipeline" } }, { @@ -3032,8 +3040,8 @@ "json": { "timestampMillis": 1705308660402, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 3, "columnCount": 3, @@ -3163,8 +3171,8 @@ "json": { "timestampMillis": 1705308660402, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 10, "columnCount": 2, @@ -3187,8 +3195,8 @@ "json": { "timestampMillis": 1716198037325, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 3, "columnCount": 3, @@ -3243,8 +3251,8 @@ "json": { "timestampMillis": 1708671016385, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 10, "columnCount": 2, diff --git a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json index 2f2a02a3352ee..65d49c59f96b8 100644 --- a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json @@ -11,7 +11,8 @@ "env": "PROD", "database": "Vmart" }, - "name": "Vmart" + "name": "Vmart", + "env": "PROD" } }, "systemMetadata": { @@ -99,7 +100,8 @@ "database": "Vmart", "schema": "public" }, - "name": "public" + "name": "public", + "env": "PROD" } }, "systemMetadata": { @@ -250,7 +252,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:46:11.762979+00:00", + "create_time": "2024-09-16 16:02:50.658164+00:00", "table_size": "0 KB" }, "name": "clicks", @@ -427,7 +429,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.748139+00:00", + "create_time": "2024-09-16 16:02:28.089168+00:00", "table_size": "2119 KB" }, "name": "customer_dimension", @@ -838,7 +840,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.756539+00:00", + "create_time": "2024-09-16 16:02:28.095005+00:00", "table_size": "145 KB" }, "name": "date_dimension", @@ -1249,7 +1251,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.761468+00:00", + "create_time": "2024-09-16 16:02:28.098768+00:00", "table_size": "327 KB" }, "name": "employee_dimension", @@ -1621,7 +1623,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.768272+00:00", + "create_time": "2024-09-16 16:02:28.103834+00:00", "table_size": "2567 KB" }, "name": "inventory_fact", @@ -1837,7 +1839,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:46:11.881911+00:00", + "create_time": "2024-09-16 16:02:50.776030+00:00", "table_size": "0 KB" }, "name": "phrases", @@ -1988,7 +1990,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.751522+00:00", + "create_time": "2024-09-16 16:02:28.091210+00:00", "table_size": "19 KB" }, "name": "product_dimension", @@ -2399,7 +2401,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.754039+00:00", + "create_time": "2024-09-16 16:02:28.093134+00:00", "table_size": "3 KB" }, "name": "promotion_dimension", @@ -2693,7 +2695,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:46:11.751917+00:00", + "create_time": "2024-09-16 16:02:50.650973+00:00", "table_size": "0 KB" }, "name": "readings", @@ -2870,7 +2872,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.763788+00:00", + "create_time": "2024-09-16 16:02:28.100511+00:00", "table_size": "1 KB" }, "name": "shipping_dimension", @@ -3060,7 +3062,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.759103+00:00", + "create_time": "2024-09-16 16:02:28.096888+00:00", "table_size": "1 KB" }, "name": "vendor_dimension", @@ -3302,7 +3304,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:46:11.212820+00:00", + "create_time": "2024-09-16 16:02:50.005433+00:00", "table_size": "0 KB" }, "name": "vmart_load_success", @@ -3453,7 +3455,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.766036+00:00", + "create_time": "2024-09-16 16:02:28.102151+00:00", "table_size": "2 KB" }, "name": "warehouse_dimension", @@ -3669,7 +3671,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:46:11.778456+00:00", + "create_time": "2024-09-16 16:02:50.666684+00:00", "table_size": "0 KB", "view_definition": "SELECT sum(customer_dimension.annual_income) AS SUM, customer_dimension.customer_state FROM public.customer_dimension WHERE (customer_dimension.customer_key IN (SELECT store_sales_fact.customer_key FROM store.store_sales_fact)) GROUP BY customer_dimension.customer_state ORDER BY customer_dimension.customer_state", "is_view": "True" @@ -3809,10 +3811,10 @@ "platform": "vertica", "env": "PROD", "database": "Vmart", - "database": "Vmart", "schema": "store" }, - "name": "store" + "name": "store", + "env": "PROD" } }, "systemMetadata": { @@ -3963,7 +3965,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.773986+00:00", + "create_time": "2024-09-16 16:02:28.107790+00:00", "table_size": "2 KB" }, "name": "store_dimension", @@ -4335,7 +4337,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.781744+00:00", + "create_time": "2024-09-16 16:02:28.113651+00:00", "table_size": "8646 KB" }, "name": "store_orders_fact", @@ -4720,7 +4722,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.776427+00:00", + "create_time": "2024-09-16 16:02:28.109623+00:00", "table_size": "225096 KB" }, "name": "store_sales_fact", @@ -5037,7 +5039,8 @@ "database": "Vmart", "schema": "online_sales" }, - "name": "online_sales" + "name": "online_sales", + "env": "PROD" } }, "systemMetadata": { @@ -5188,7 +5191,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.789350+00:00", + "create_time": "2024-09-16 16:02:28.119351+00:00", "table_size": "6 KB" }, "name": "call_center_dimension", @@ -5482,7 +5485,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.786989+00:00", + "create_time": "2024-09-16 16:02:28.117625+00:00", "table_size": "9 KB" }, "name": "online_page_dimension", @@ -5698,7 +5701,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-18 12:45:49.791761+00:00", + "create_time": "2024-09-16 16:02:28.121084+00:00", "table_size": "182385 KB" }, "name": "online_sales_fact", @@ -6052,13 +6055,13 @@ "platform": "vertica", "env": "PROD", "database": "Vmart", - "database": "Vmart", "cluster_type": "Enterprise", - "cluster_size": "243 GB", + "cluster_size": "103 GB", "subcluster": " ", "communal_storage_path": "" }, - "name": "Vmart" + "name": "Vmart", + "env": "PROD" } }, "systemMetadata": { @@ -6131,9 +6134,10 @@ "schema": "public", "projection_count": "12", "udx_list": "APPROXIMATE_COUNT_DISTINCT_SYNOPSIS_INFO, APPROXIMATE_MEDIAN, APPROXIMATE_PERCENTILE, AcdDataToCount, AcdDataToLongSyn, AcdDataToSyn, AcdSynToCount, AcdSynToSyn, DelimitedExport, DelimitedExportMulti, EmptyMap, Explode, FAvroParser, FCefParser, FCsvParser, FDelimitedPairParser, FDelimitedParser, FIDXParser, FJSONParser, FRegexParser, FlexTokenizer, JsonExport, JsonExportMulti, KafkaAvroParser, KafkaCheckBrokers, KafkaExport, KafkaInsertDelimiters, KafkaInsertLengths, KafkaJsonParser, KafkaListManyTopics, KafkaListTopics, KafkaOffsets, KafkaParser, KafkaSource, KafkaTopicDetails, MSE, MapAggregate, MapAggregate, MapContainsKey, MapContainsKey, MapContainsValue, MapContainsValue, MapDelimitedExtractor, MapItems, MapItems, MapJSONExtractor, MapKeys, MapKeys, MapKeysInfo, MapKeysInfo, MapLookup, MapLookup, MapLookup, MapPut, MapRegexExtractor, MapSize, MapSize, MapToString, MapToString, MapValues, MapValues, MapValuesOrField, MapVersion, MapVersion, OrcExport, OrcExportMulti, PRC, ParquetExport, ParquetExportMulti, PickBestType, PickBestType, PickBestType, ROC, STV_AsGeoJSON, STV_AsGeoJSON, STV_AsGeoJSON, STV_Create_Index, STV_Create_Index, STV_Create_Index, STV_DWithin, STV_DWithin, STV_DWithin, STV_Describe_Index, STV_Drop_Index, STV_Export2Shapefile, STV_Extent, STV_Extent, STV_ForceLHR, STV_Geography, STV_Geography, STV_GeographyPoint, STV_Geometry, STV_Geometry, STV_GeometryPoint, STV_GeometryPoint, STV_GetExportShapefileDirectory, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_IsValidReason, STV_IsValidReason, STV_IsValidReason, STV_LineStringPoint, STV_LineStringPoint, STV_LineStringPoint, STV_MemSize, STV_MemSize, STV_MemSize, STV_NN, STV_NN, STV_NN, STV_PolygonPoint, STV_PolygonPoint, STV_PolygonPoint, STV_Refresh_Index, STV_Refresh_Index, STV_Refresh_Index, STV_Rename_Index, STV_Reverse, STV_SetExportShapefileDirectory, STV_ShpCreateTable, STV_ShpParser, STV_ShpSource, ST_Area, ST_Area, ST_Area, ST_AsBinary, ST_AsBinary, ST_AsBinary, ST_AsText, ST_AsText, ST_AsText, ST_Boundary, ST_Buffer, ST_Centroid, ST_Contains, ST_Contains, ST_Contains, ST_ConvexHull, ST_Crosses, ST_Difference, ST_Disjoint, ST_Disjoint, ST_Disjoint, ST_Distance, ST_Distance, ST_Distance, ST_Envelope, ST_Equals, ST_Equals, ST_Equals, ST_GeoHash, ST_GeoHash, ST_GeoHash, ST_GeographyFromText, ST_GeographyFromWKB, ST_GeomFromGeoHash, ST_GeomFromGeoJSON, ST_GeomFromGeoJSON, ST_GeomFromText, ST_GeomFromText, ST_GeomFromWKB, ST_GeomFromWKB, ST_GeometryN, ST_GeometryN, ST_GeometryN, ST_GeometryType, ST_GeometryType, ST_GeometryType, ST_Intersection, ST_Intersects, ST_Intersects, ST_IsEmpty, ST_IsEmpty, ST_IsEmpty, ST_IsSimple, ST_IsSimple, ST_IsSimple, ST_IsValid, ST_IsValid, ST_IsValid, ST_Length, ST_Length, ST_Length, ST_NumGeometries, ST_NumGeometries, ST_NumGeometries, ST_NumPoints, ST_NumPoints, ST_NumPoints, ST_Overlaps, ST_PointFromGeoHash, ST_PointN, ST_PointN, ST_PointN, ST_Relate, ST_SRID, ST_SRID, ST_SRID, ST_Simplify, ST_SimplifyPreserveTopology, ST_SymDifference, ST_Touches, ST_Touches, ST_Touches, ST_Transform, ST_Union, ST_Union, ST_Within, ST_Within, ST_Within, ST_X, ST_X, ST_X, ST_XMax, ST_XMax, ST_XMax, ST_XMin, ST_XMin, ST_XMin, ST_Y, ST_Y, ST_Y, ST_YMax, ST_YMax, ST_YMax, ST_YMin, ST_YMin, ST_YMin, ST_intersects, SetMapKeys, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_NumCol, Unnest, VoltageSecureAccess, VoltageSecureAccess, VoltageSecureConfigure, VoltageSecureConfigureGlobal, VoltageSecureProtect, VoltageSecureProtect, VoltageSecureProtectAllKeys, VoltageSecureRefreshPolicy, VoltageSecureVersion, append_centers, apply_bisecting_kmeans, apply_iforest, apply_inverse_pca, apply_inverse_svd, apply_kmeans, apply_kprototypes, apply_normalize, apply_one_hot_encoder, apply_pca, apply_svd, approximate_quantiles, ar_create_blobs, ar_final_newton, ar_save_model, ar_transition_newton, arima_bfgs, arima_line_search, arima_save_model, avg_all_columns_local, bisecting_kmeans_init_model, bk_apply_best_kmeans_results, bk_compute_totss_local, bk_finalize_model, bk_get_rows_in_active_cluster, bk_kmeans_compute_local_centers, bk_kmeans_compute_withinss, bk_kmeans_fast_random_init, bk_kmeans_slow_random_init, bk_kmeanspp_init_cur_cluster, bk_kmeanspp_reset_blob, bk_kmeanspp_select_new_centers, bk_kmeanspp_within_chunk_sum, bk_save_final_model, bk_write_new_cluster_level, blob_to_table, bufUdx, bufUdx, calc_pseudo_centers, calculate_alpha_linear, calculate_hessian_linear1, calculate_hessian_linear2, chi_squared, cleanup_kmeans_files, compute_and_save_global_center, compute_and_save_new_centers, compute_local_totss, compute_local_withinss, compute_new_local_centers, confusion_matrix, coordinate_descent_covariance, corr_matrix, count_rows_in_blob, create_aggregator_blob, error_rate, evaluate_naive_bayes_model, evaluate_reg_model, evaluate_svm_model, export_model_files, finalize_blob_resource_group, get_attr_minmax, get_attr_robust_zscore, get_attr_zscore, get_model_attribute, get_model_summary, get_robust_zscore_median, iforest_create_blobs, iforest_phase0_udf1, iforest_phase0_udf2, iforest_phase1_udf1, iforest_phase1_udf2, iforest_phase1_udf3, iforest_phase1_udf4, iforest_phase2_udf1, iforest_phase2_udf2, iforest_phase2_udf3, iforest_phase2_udf4, iforest_save_model, import_model_files, isOrContains, kmeansAddMetricsToModel, kmeans_init_blobs, kmeans_to_write_final_centers, lift_table, line_search_logistic1, line_search_logistic2, load_rows_into_blocks, map_factor, math_op, matrix_global_xtx, matrix_local_xtx, mode_finder, model_converter, naive_bayes_phase1, naive_bayes_phase1_blob, naive_bayes_phase2, pca_prep1_global, pca_prep1_local, pca_prep2, pmml_parser, predict_arima, predict_autoregressor, predict_linear_reg, predict_logistic_reg, predict_moving_average, predict_naive_bayes, predict_naive_bayes_classes, predict_pmml, predict_poisson_reg, predict_rf_classifier, predict_rf_classifier_classes, predict_rf_regressor, predict_svm_classifier, predict_svm_regressor, predict_xgb_classifier, predict_xgb_classifier_classes, predict_xgb_regressor, random_init, random_init_write, read_from_dfblob, read_map_factor, read_ptree, read_tree, reg_final_bfgs, reg_final_newton, reg_transition_bfgs, reg_transition_newton, reg_write_model, remove_blob, reverse_normalize, rf_blob, rf_clean, rf_phase0_udf1, rf_phase0_udf2, rf_phase1_udf1, rf_phase1_udf2, rf_phase1_udf3, rf_phase1_udf4, rf_phase2_udf1, rf_phase2_udf2, rf_phase2_udf3, rf_phase2_udf4, rf_predictor_importance, rf_save_model, rsquared, save_cv_result, save_pca_model, save_svd_model, save_svm_model, select_new_centers, store_minmax_model, store_one_hot_encoder_model, store_robust_zscore_model, store_zscore_model, table_to_blob, table_to_dfblob, tokenize, topk, update_and_return_sum_of_squared_distances, upgrade_model_format, writeInitialKmeansModelToDfs, xgb_create_blobs, xgb_phase0_udf1, xgb_phase0_udf2, xgb_phase1_udf1, xgb_phase1_udf2, xgb_phase1_udf3, xgb_phase2_udf1, xgb_phase2_udf2, xgb_phase2_udf3, xgb_predictor_importance, xgb_prune, xgb_save_model, yule_walker, ", - "udx_language": "ApproximateLib -- Approximate package | FlexTableLib -- Flexible Tables Data Load and Query | OrcExportLib -- Orc export package | JsonExportLib -- Json data export package | PlaceLib -- Geospatial package | ParquetExportLib -- Parquet export package | ComplexTypesLib -- Functions for Complex Types | VoltageSecureLib -- Voltage SecureData Connector | KafkaLib -- Kafka streaming load and export | MachineLearningLib -- Machine learning package | DelimitedExportLib -- Delimited data export package | TransformFunctions -- User-defined Python library | " + "udx_language": "ParquetExportLib -- Parquet export package | DelimitedExportLib -- Delimited data export package | ApproximateLib -- Approximate package | ComplexTypesLib -- Functions for Complex Types | FlexTableLib -- Flexible Tables Data Load and Query | JsonExportLib -- Json data export package | PlaceLib -- Geospatial package | VoltageSecureLib -- Voltage SecureData Connector | MachineLearningLib -- Machine learning package | OrcExportLib -- Orc export package | KafkaLib -- Kafka streaming load and export | TransformFunctions -- User-defined Python library | " }, - "name": "public" + "name": "public", + "env": "PROD" } }, "systemMetadata": { @@ -9538,7 +9542,8 @@ "udx_list": "", "udx_language": "" }, - "name": "store" + "name": "store", + "env": "PROD" } }, "systemMetadata": { @@ -10763,7 +10768,8 @@ "udx_list": "", "udx_language": "" }, - "name": "online_sales" + "name": "online_sales", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json index 49c24d87105bb..60085f061a653 100644 --- a/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json @@ -12,7 +12,8 @@ "database": "test-database" }, "name": "test-database", - "qualifiedName": "arn:aws:glue:eu-east-1:123412341234:database/test-database" + "qualifiedName": "arn:aws:glue:eu-east-1:123412341234:database/test-database", + "env": "PROD" } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json index 01ecdbb4fd331..46f3b772e641b 100644 --- a/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json @@ -14,7 +14,8 @@ "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "delta-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database" + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database", + "env": "PROD" } } }, diff --git a/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json index e71ede4f370ff..7ddbe47aaaf16 100644 --- a/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json @@ -14,7 +14,8 @@ "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "delta-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database" + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database", + "env": "PROD" } } }, diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json index 77e8354e657a6..87971de12fbb3 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json @@ -1,1342 +1,1345 @@ [ - { - "entityType": "container", - "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "glue", - "env": "PROD", - "database": "flights-database", - "param1": "value1", - "param2": "value2", - "LocationUri": "s3://test-bucket/test-prefix", - "CreateTime": "June 09, 2021 at 14:14:19" - }, - "name": "flights-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database" - } +{ + "entityType": "container", + "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "glue", + "env": "PROD", + "database": "flights-database", + "param1": "value1", + "param2": "value2", + "LocationUri": "s3://test-bucket/test-prefix", + "CreateTime": "June 09, 2021 at 14:14:19" + }, + "name": "flights-database", + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database", + "env": "PROD" } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:glue" - } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:glue" } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Database" - ] - } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "glue", - "env": "PROD", - "database": "test-database", - "CreateTime": "June 01, 2021 at 14:55:02" - }, - "name": "test-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database" - } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "glue", + "env": "PROD", + "database": "test-database", + "CreateTime": "June 01, 2021 at 14:55:02" + }, + "name": "test-database", + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database", + "env": "PROD" } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:glue" - } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:glue" } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Database" - ] - } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "glue", - "env": "PROD", - "database": "empty-database", - "CreateTime": "June 01, 2021 at 14:55:13" - }, - "name": "empty-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database" - } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "glue", + "env": "PROD", + "database": "empty-database", + "CreateTime": "June 01, 2021 at 14:55:13" + }, + "name": "empty-database", + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database", + "env": "PROD" } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:glue" - } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:glue" } - }, - { - "entityType": "container", - "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Database" - ] - } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "CrawlerSchemaDeserializerVersion": "1.0", - "CrawlerSchemaSerializerVersion": "1.0", - "UPDATED_BY_CRAWLER": "flights-crawler", - "averageRecordSize": "55", - "avro.schema.literal": "{\"type\":\"record\",\"name\":\"flights_avro_subset\",\"namespace\":\"default\",\"fields\":[{\"name\":\"yr\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"flightdate\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"uniquecarrier\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"airlineid\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"carrier\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"flightnum\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"origin\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"dest\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"depdelay\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"carrierdelay\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"weatherdelay\",\"type\":[\"null\",\"int\"],\"default\":null}]}", - "classification": "avro", - "compressionType": "none", - "objectCount": "30", - "recordCount": "169222196", - "sizeKey": "9503351413", - "typeOfData": "file", - "Location": "s3://crawler-public-us-west-2/flight/avro/", - "InputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat", - "OutputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat", - "Compressed": "False", - "NumberOfBuckets": "-1", - "SerdeInfo": "{'SerializationLibrary': 'org.apache.hadoop.hive.serde2.avro.AvroSerDe', 'Parameters': {'avro.schema.literal': '{\"type\":\"record\",\"name\":\"flights_avro_subset\",\"namespace\":\"default\",\"fields\":[{\"name\":\"yr\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"flightdate\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"uniquecarrier\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"airlineid\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"carrier\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"flightnum\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"origin\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"dest\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"depdelay\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"carrierdelay\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"weatherdelay\",\"type\":[\"null\",\"int\"],\"default\":null}]}', 'serialization.format': '1'}}", - "BucketColumns": "[]", - "SortColumns": "[]", - "StoredAsSubDirectories": "False" - }, - "name": "avro", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:table/flights-database/avro", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "flights-database.avro", - "platform": "urn:li:dataPlatform:glue", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].yr", - "nullable": true, - "description": "test comment", + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "CrawlerSchemaDeserializerVersion": "1.0", + "CrawlerSchemaSerializerVersion": "1.0", + "UPDATED_BY_CRAWLER": "flights-crawler", + "averageRecordSize": "55", + "avro.schema.literal": "{\"type\":\"record\",\"name\":\"flights_avro_subset\",\"namespace\":\"default\",\"fields\":[{\"name\":\"yr\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"flightdate\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"uniquecarrier\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"airlineid\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"carrier\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"flightnum\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"origin\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"dest\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"depdelay\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"carrierdelay\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"weatherdelay\",\"type\":[\"null\",\"int\"],\"default\":null}]}", + "classification": "avro", + "compressionType": "none", + "objectCount": "30", + "recordCount": "169222196", + "sizeKey": "9503351413", + "typeOfData": "file", + "Location": "s3://crawler-public-us-west-2/flight/avro/", + "InputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat", + "OutputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat", + "Compressed": "False", + "NumberOfBuckets": "-1", + "SerdeInfo": "{'SerializationLibrary': 'org.apache.hadoop.hive.serde2.avro.AvroSerDe', 'Parameters': {'avro.schema.literal': '{\"type\":\"record\",\"name\":\"flights_avro_subset\",\"namespace\":\"default\",\"fields\":[{\"name\":\"yr\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"flightdate\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"uniquecarrier\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"airlineid\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"carrier\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"flightnum\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"origin\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"dest\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"depdelay\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"carrierdelay\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"weatherdelay\",\"type\":[\"null\",\"int\"],\"default\":null}]}', 'serialization.format': '1'}}", + "BucketColumns": "[]", + "SortColumns": "[]", + "StoredAsSubDirectories": "False" + }, + "name": "avro", + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:table/flights-database/avro", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "flights-database.avro", + "platform": "urn:li:dataPlatform:glue", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].yr", + "nullable": true, + "description": "test comment", + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=string].flightdate", - "nullable": true, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].flightdate", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.StringType": {} + } }, - { - "fieldPath": "[version=2.0].[type=string].uniquecarrier", - "nullable": true, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].uniquecarrier", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.StringType": {} + } }, - { - "fieldPath": "[version=2.0].[type=int].airlineid", - "nullable": true, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=int].airlineid", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=string].carrier", - "nullable": true, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].carrier", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.StringType": {} + } }, - { - "fieldPath": "[version=2.0].[type=string].flightnum", - "nullable": true, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].flightnum", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.StringType": {} + } }, - { - "fieldPath": "[version=2.0].[type=string].origin", - "nullable": true, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].origin", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.StringType": {} + } }, - { - "fieldPath": "[version=2.0].[type=string].year", - "nullable": true, - "description": "partition test comment", + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].year", + "nullable": true, + "description": "partition test comment", + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:glue" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:owner", - "type": "DATAOWNER" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:baz:bob" + "com.linkedin.pegasus2avro.schema.StringType": {} + } }, - { - "tag": "urn:li:tag:foo:bar" - } - ] + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:glue" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:owner", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" } } - ] - } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:baz:bob" + }, + { + "tag": "urn:li:tag:foo:bar" + } + ] + } + } + ] } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba" - } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba" } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "CrawlerSchemaDeserializerVersion": "1.0", - "CrawlerSchemaSerializerVersion": "1.0", - "UPDATED_BY_CRAWLER": "test-jsons", - "averageRecordSize": "273", - "classification": "json", - "compressionType": "none", - "objectCount": "1", - "recordCount": "1", - "sizeKey": "273", - "typeOfData": "file", - "Location": "s3://test-glue-jsons/markers/", - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "Compressed": "False", - "NumberOfBuckets": "-1", - "SerdeInfo": "{'SerializationLibrary': 'org.openx.data.jsonserde.JsonSerDe', 'Parameters': {'paths': 'markers'}}", - "BucketColumns": "[]", - "SortColumns": "[]", - "StoredAsSubDirectories": "False" - }, - "name": "test_jsons_markers", - "qualifiedName": "arn:aws:glue:us-west-2:795586375822:table/test-database/test_jsons_markers", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test-database.test_jsons_markers", - "platform": "urn:li:dataPlatform:glue", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].markers", - "nullable": true, + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "CrawlerSchemaDeserializerVersion": "1.0", + "CrawlerSchemaSerializerVersion": "1.0", + "UPDATED_BY_CRAWLER": "test-jsons", + "averageRecordSize": "273", + "classification": "json", + "compressionType": "none", + "objectCount": "1", + "recordCount": "1", + "sizeKey": "273", + "typeOfData": "file", + "Location": "s3://test-glue-jsons/markers/", + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "Compressed": "False", + "NumberOfBuckets": "-1", + "SerdeInfo": "{'SerializationLibrary': 'org.openx.data.jsonserde.JsonSerDe', 'Parameters': {'paths': 'markers'}}", + "BucketColumns": "[]", + "SortColumns": "[]", + "StoredAsSubDirectories": "False" + }, + "name": "test_jsons_markers", + "qualifiedName": "arn:aws:glue:us-west-2:795586375822:table/test-database/test_jsons_markers", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test-database.test_jsons_markers", + "platform": "urn:li:dataPlatform:glue", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].markers", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "record" - ] - } + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] } - }, - "nativeDataType": "array,location:array>>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array,location:array>>\"}" + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].markers.[type=string].name", - "nullable": true, + "nativeDataType": "array,location:array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array,location:array>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].markers.[type=string].name", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.StringType": {} + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].markers.[type=array].[type=double].position", - "nullable": true, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].markers.[type=array].[type=double].position", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "double" - ] - } + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "double" + ] } - }, - "nativeDataType": "array", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array\"}" + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].markers.[type=array].[type=double].location", - "nullable": true, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].markers.[type=array].[type=double].location", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "double" - ] - } + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "double" + ] } - }, - "nativeDataType": "array", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:glue" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:owner", - "type": "DATAOWNER" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:baz:bob" + } }, - { - "tag": "urn:li:tag:foo:bar" - } - ] + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:glue" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:owner", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" } } - ] - } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:baz:bob" + }, + { + "tag": "urn:li:tag:foo:bar" + } + ] + } + } + ] } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7" - } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7" } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "CrawlerSchemaDeserializerVersion": "1.0", - "CrawlerSchemaSerializerVersion": "1.0", - "UPDATED_BY_CRAWLER": "test", - "averageRecordSize": "19", - "classification": "parquet", - "compressionType": "none", - "objectCount": "60", - "recordCount": "167497743", - "sizeKey": "4463574900", - "typeOfData": "file", - "Location": "s3://crawler-public-us-west-2/flight/parquet/", - "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", - "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", - "Compressed": "False", - "NumberOfBuckets": "-1", - "SerdeInfo": "{'SerializationLibrary': 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'Parameters': {'serialization.format': '1'}}", - "BucketColumns": "[]", - "SortColumns": "[]", - "StoredAsSubDirectories": "False" - }, - "name": "test_parquet", - "qualifiedName": "arn:aws:glue:us-west-2:795586375822:table/test-database/test_parquet", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "test-database.test_parquet", - "platform": "urn:li:dataPlatform:glue", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].yr", - "nullable": true, + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "CrawlerSchemaDeserializerVersion": "1.0", + "CrawlerSchemaSerializerVersion": "1.0", + "UPDATED_BY_CRAWLER": "test", + "averageRecordSize": "19", + "classification": "parquet", + "compressionType": "none", + "objectCount": "60", + "recordCount": "167497743", + "sizeKey": "4463574900", + "typeOfData": "file", + "Location": "s3://crawler-public-us-west-2/flight/parquet/", + "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", + "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", + "Compressed": "False", + "NumberOfBuckets": "-1", + "SerdeInfo": "{'SerializationLibrary': 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'Parameters': {'serialization.format': '1'}}", + "BucketColumns": "[]", + "SortColumns": "[]", + "StoredAsSubDirectories": "False" + }, + "name": "test_parquet", + "qualifiedName": "arn:aws:glue:us-west-2:795586375822:table/test-database/test_parquet", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test-database.test_parquet", + "platform": "urn:li:dataPlatform:glue", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].yr", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=int].quarter", - "nullable": true, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=int].quarter", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=int].month", - "nullable": true, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=int].month", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=int].dayofmonth", - "nullable": true, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=int].dayofmonth", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + "com.linkedin.pegasus2avro.schema.NumberType": {} + } }, - { - "fieldPath": "[version=2.0].[type=string].year", - "nullable": true, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].year", + "nullable": true, + "type": { "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.DataPlatformInstance": { - "platform": "urn:li:dataPlatform:glue" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:owner", - "type": "DATAOWNER" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:baz:bob" + "com.linkedin.pegasus2avro.schema.StringType": {} + } }, - { - "tag": "urn:li:tag:foo:bar" - } - ] + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:glue" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:owner", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" } } - ] - } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:baz:bob" + }, + { + "tag": "urn:li:tag:foo:bar" + } + ] + } + } + ] } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Table" - ] - } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] } - }, - { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7" - } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7" } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(glue,test-job-1,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "customProperties": { - "role": "arn:aws:iam::123412341234:role/service-role/AWSGlueServiceRole-glue-crawler", - "created": "2021-06-10 16:51:25.690000", - "modified": "2021-06-10 16:55:35.307000", - "command": "s3://aws-glue-assets-123412341234-us-west-2/scripts/job-1.py" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", - "name": "test-job-1", - "description": "The first test job" - } + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(glue,test-job-1,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "customProperties": { + "role": "arn:aws:iam::123412341234:role/service-role/AWSGlueServiceRole-glue-crawler", + "created": "2021-06-10 16:51:25.690000", + "modified": "2021-06-10 16:55:35.307000", + "command": "s3://aws-glue-assets-123412341234-us-west-2/scripts/job-1.py" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1", + "description": "The first test job" } - ] - } + } + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(glue,test-job-2,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "customProperties": { - "role": "arn:aws:iam::123412341234:role/service-role/AWSGlueServiceRole-glue-crawler", - "created": "2021-06-10 16:58:32.469000", - "modified": "2021-06-10 16:58:32.469000", - "command": "s3://aws-glue-assets-123412341234-us-west-2/scripts/job-2.py" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", - "name": "test-job-2", - "description": "The second test job" - } + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(glue,test-job-2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "customProperties": { + "role": "arn:aws:iam::123412341234:role/service-role/AWSGlueServiceRole-glue-crawler", + "created": "2021-06-10 16:58:32.469000", + "modified": "2021-06-10 16:58:32.469000", + "command": "s3://aws-glue-assets-123412341234-us-west-2/scripts/job-2.py" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", + "name": "test-job-2", + "description": "The second test job" } - ] - } + } + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "f": "lambda row : ()", - "transformation_ctx": "\"Transform0\"", - "transformType": "Filter", - "nodeId": "Transform0_job1" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", - "name": "test-job-1:Filter-Transform0_job1", - "type": { - "string": "GLUE" - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [], - "outputDatasets": [], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)" - ] + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "f": "lambda row : ()", + "transformation_ctx": "\"Transform0\"", + "transformType": "Filter", + "nodeId": "Transform0_job1" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1:Filter-Transform0_job1", + "type": { + "string": "GLUE" } } - ] - } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)" + ] + } + } + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform1_job1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "mappings": "[(\"yr\", \"int\", \"yr\", \"int\"), (\"flightdate\", \"string\", \"flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"airlineid\", \"int\"), (\"carrier\", \"string\", \"carrier\", \"string\"), (\"flightnum\", \"string\", \"flightnum\", \"string\"), (\"origin\", \"string\", \"origin\", \"string\"), (\"dest\", \"string\", \"dest\", \"string\"), (\"depdelay\", \"int\", \"depdelay\", \"int\"), (\"carrierdelay\", \"int\", \"carrierdelay\", \"int\"), (\"weatherdelay\", \"int\", \"weatherdelay\", \"int\"), (\"year\", \"string\", \"year\", \"string\")]", - "transformation_ctx": "\"Transform1\"", - "transformType": "ApplyMapping", - "nodeId": "Transform1_job1" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", - "name": "test-job-1:ApplyMapping-Transform1_job1", - "type": { - "string": "GLUE" - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [], - "outputDatasets": [], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)" - ] + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform1_job1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "mappings": "[(\"yr\", \"int\", \"yr\", \"int\"), (\"flightdate\", \"string\", \"flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"airlineid\", \"int\"), (\"carrier\", \"string\", \"carrier\", \"string\"), (\"flightnum\", \"string\", \"flightnum\", \"string\"), (\"origin\", \"string\", \"origin\", \"string\"), (\"dest\", \"string\", \"dest\", \"string\"), (\"depdelay\", \"int\", \"depdelay\", \"int\"), (\"carrierdelay\", \"int\", \"carrierdelay\", \"int\"), (\"weatherdelay\", \"int\", \"weatherdelay\", \"int\"), (\"year\", \"string\", \"year\", \"string\")]", + "transformation_ctx": "\"Transform1\"", + "transformType": "ApplyMapping", + "nodeId": "Transform1_job1" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1:ApplyMapping-Transform1_job1", + "type": { + "string": "GLUE" } } - ] - } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)" + ] + } + } + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "mappings": "[(\"yr\", \"int\", \"yr\", \"int\"), (\"flightdate\", \"string\", \"flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"airlineid\", \"int\"), (\"carrier\", \"string\", \"carrier\", \"string\"), (\"flightnum\", \"string\", \"flightnum\", \"string\"), (\"origin\", \"string\", \"origin\", \"string\"), (\"dest\", \"string\", \"dest\", \"string\"), (\"depdelay\", \"int\", \"depdelay\", \"int\"), (\"carrierdelay\", \"int\", \"carrierdelay\", \"int\"), (\"weatherdelay\", \"int\", \"weatherdelay\", \"int\"), (\"year\", \"string\", \"year\", \"string\")]", - "transformation_ctx": "\"Transform2\"", - "transformType": "ApplyMapping", - "nodeId": "Transform2_job1" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", - "name": "test-job-1:ApplyMapping-Transform2_job1", - "type": { - "string": "GLUE" - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)" - ], - "outputDatasets": [], - "inputDatajobs": [] + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "mappings": "[(\"yr\", \"int\", \"yr\", \"int\"), (\"flightdate\", \"string\", \"flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"airlineid\", \"int\"), (\"carrier\", \"string\", \"carrier\", \"string\"), (\"flightnum\", \"string\", \"flightnum\", \"string\"), (\"origin\", \"string\", \"origin\", \"string\"), (\"dest\", \"string\", \"dest\", \"string\"), (\"depdelay\", \"int\", \"depdelay\", \"int\"), (\"carrierdelay\", \"int\", \"carrierdelay\", \"int\"), (\"weatherdelay\", \"int\", \"weatherdelay\", \"int\"), (\"year\", \"string\", \"year\", \"string\")]", + "transformation_ctx": "\"Transform2\"", + "transformType": "ApplyMapping", + "nodeId": "Transform2_job1" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1:ApplyMapping-Transform2_job1", + "type": { + "string": "GLUE" } } - ] - } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [] + } + } + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Join-Transform3_job1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "keys2": "[\"(right) flightdate\"]", - "transformation_ctx": "\"Transform3\"", - "keys1": "[\"yr\"]", - "transformType": "Join", - "nodeId": "Transform3_job1" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", - "name": "test-job-1:Join-Transform3_job1", - "type": { - "string": "GLUE" - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [], - "outputDatasets": [], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)" - ] + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Join-Transform3_job1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "keys2": "[\"(right) flightdate\"]", + "transformation_ctx": "\"Transform3\"", + "keys1": "[\"yr\"]", + "transformType": "Join", + "nodeId": "Transform3_job1" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1:Join-Transform3_job1", + "type": { + "string": "GLUE" } } - ] - } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)" + ] + } + } + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "mappings": "[(\"yr\", \"int\", \"yr\", \"int\"), (\"flightdate\", \"string\", \"flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"airlineid\", \"int\"), (\"carrier\", \"string\", \"carrier\", \"string\"), (\"flightnum\", \"string\", \"flightnum\", \"string\"), (\"origin\", \"string\", \"origin\", \"string\"), (\"dest\", \"string\", \"dest\", \"string\"), (\"depdelay\", \"int\", \"depdelay\", \"int\"), (\"carrierdelay\", \"int\", \"carrierdelay\", \"int\"), (\"weatherdelay\", \"int\", \"weatherdelay\", \"int\"), (\"year\", \"string\", \"year\", \"string\")]", - "transformation_ctx": "\"Transform4\"", - "transformType": "ApplyMapping", - "nodeId": "Transform4_job1" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", - "name": "test-job-1:ApplyMapping-Transform4_job1", - "type": { - "string": "GLUE" - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [], - "outputDatasets": [], - "inputDatajobs": [] + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "mappings": "[(\"yr\", \"int\", \"yr\", \"int\"), (\"flightdate\", \"string\", \"flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"airlineid\", \"int\"), (\"carrier\", \"string\", \"carrier\", \"string\"), (\"flightnum\", \"string\", \"flightnum\", \"string\"), (\"origin\", \"string\", \"origin\", \"string\"), (\"dest\", \"string\", \"dest\", \"string\"), (\"depdelay\", \"int\", \"depdelay\", \"int\"), (\"carrierdelay\", \"int\", \"carrierdelay\", \"int\"), (\"weatherdelay\", \"int\", \"weatherdelay\", \"int\"), (\"year\", \"string\", \"year\", \"string\")]", + "transformation_ctx": "\"Transform4\"", + "transformType": "ApplyMapping", + "nodeId": "Transform4_job1" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1:ApplyMapping-Transform4_job1", + "type": { + "string": "GLUE" } } - ] - } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [] + } + } + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform5_job1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "mappings": "[(\"yr\", \"int\", \"(right) yr\", \"int\"), (\"flightdate\", \"string\", \"(right) flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"(right) uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"(right) airlineid\", \"int\"), (\"carrier\", \"string\", \"(right) carrier\", \"string\"), (\"flightnum\", \"string\", \"(right) flightnum\", \"string\"), (\"origin\", \"string\", \"(right) origin\", \"string\"), (\"dest\", \"string\", \"(right) dest\", \"string\"), (\"depdelay\", \"int\", \"(right) depdelay\", \"int\"), (\"carrierdelay\", \"int\", \"(right) carrierdelay\", \"int\"), (\"weatherdelay\", \"int\", \"(right) weatherdelay\", \"int\"), (\"year\", \"string\", \"(right) year\", \"string\")]", - "transformation_ctx": "\"Transform5\"", - "transformType": "ApplyMapping", - "nodeId": "Transform5_job1" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", - "name": "test-job-1:ApplyMapping-Transform5_job1", - "type": { - "string": "GLUE" - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [], - "outputDatasets": [], - "inputDatajobs": [] + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform5_job1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "mappings": "[(\"yr\", \"int\", \"(right) yr\", \"int\"), (\"flightdate\", \"string\", \"(right) flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"(right) uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"(right) airlineid\", \"int\"), (\"carrier\", \"string\", \"(right) carrier\", \"string\"), (\"flightnum\", \"string\", \"(right) flightnum\", \"string\"), (\"origin\", \"string\", \"(right) origin\", \"string\"), (\"dest\", \"string\", \"(right) dest\", \"string\"), (\"depdelay\", \"int\", \"(right) depdelay\", \"int\"), (\"carrierdelay\", \"int\", \"(right) carrierdelay\", \"int\"), (\"weatherdelay\", \"int\", \"(right) weatherdelay\", \"int\"), (\"year\", \"string\", \"(right) year\", \"string\")]", + "transformation_ctx": "\"Transform5\"", + "transformType": "ApplyMapping", + "nodeId": "Transform5_job1" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1:ApplyMapping-Transform5_job1", + "type": { + "string": "GLUE" } } - ] - } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [] + } + } + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "connection_type": "s3", - "format": "json", - "connection_options": "{'path': 's3://test-glue-jsons/', 'partitionKeys': []}", - "transformation_ctx": "DataSink1" - }, - "tags": [] - } + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "connection_type": "s3", + "format": "json", + "connection_options": "{'path': 's3://test-glue-jsons/', 'partitionKeys': []}", + "transformation_ctx": "DataSink1" + }, + "tags": [] } - ] - } + } + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SplitFields-Transform0_job2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "paths": "[\"yr\", \"quarter\", \"month\", \"dayofmonth\", \"dayofweek\", \"flightdate\", \"uniquecarrier\"]", - "name2": "\"Transform0Output1\"", - "name1": "\"Transform0Output0\"", - "transformation_ctx": "\"Transform0\"", - "transformType": "SplitFields", - "nodeId": "Transform0_job2" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", - "name": "test-job-2:SplitFields-Transform0_job2", - "type": { - "string": "GLUE" - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [], - "outputDatasets": [], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)" - ] + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SplitFields-Transform0_job2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "paths": "[\"yr\", \"quarter\", \"month\", \"dayofmonth\", \"dayofweek\", \"flightdate\", \"uniquecarrier\"]", + "name2": "\"Transform0Output1\"", + "name1": "\"Transform0Output0\"", + "transformation_ctx": "\"Transform0\"", + "transformType": "SplitFields", + "nodeId": "Transform0_job2" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", + "name": "test-job-2:SplitFields-Transform0_job2", + "type": { + "string": "GLUE" } } - ] - } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)" + ] + } + } + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "mappings": "[(\"yr\", \"int\", \"yr\", \"int\"), (\"quarter\", \"int\", \"quarter\", \"int\"), (\"month\", \"int\", \"month\", \"int\"), (\"dayofmonth\", \"int\", \"dayofmonth\", \"int\"), (\"dayofweek\", \"int\", \"dayofweek\", \"int\"), (\"flightdate\", \"string\", \"flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"airlineid\", \"int\"), (\"carrier\", \"string\", \"carrier\", \"string\")]", - "transformation_ctx": "\"Transform1\"", - "transformType": "ApplyMapping", - "nodeId": "Transform1_job2" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", - "name": "test-job-2:ApplyMapping-Transform1_job2", - "type": { - "string": "GLUE" - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)" - ], - "outputDatasets": [], - "inputDatajobs": [] + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "mappings": "[(\"yr\", \"int\", \"yr\", \"int\"), (\"quarter\", \"int\", \"quarter\", \"int\"), (\"month\", \"int\", \"month\", \"int\"), (\"dayofmonth\", \"int\", \"dayofmonth\", \"int\"), (\"dayofweek\", \"int\", \"dayofweek\", \"int\"), (\"flightdate\", \"string\", \"flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"airlineid\", \"int\"), (\"carrier\", \"string\", \"carrier\", \"string\")]", + "transformation_ctx": "\"Transform1\"", + "transformType": "ApplyMapping", + "nodeId": "Transform1_job2" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", + "name": "test-job-2:ApplyMapping-Transform1_job2", + "type": { + "string": "GLUE" } } - ] - } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [] + } + } + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "missing_values_column": "\"dayofmonth\"", - "transformation_ctx": "\"Transform2\"", - "transformType": "FillMissingValues", - "nodeId": "Transform2_job2" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", - "name": "test-job-2:FillMissingValues-Transform2_job2", - "type": { - "string": "GLUE" - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [], - "outputDatasets": [], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)" - ] + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "missing_values_column": "\"dayofmonth\"", + "transformation_ctx": "\"Transform2\"", + "transformType": "FillMissingValues", + "nodeId": "Transform2_job2" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", + "name": "test-job-2:FillMissingValues-Transform2_job2", + "type": { + "string": "GLUE" } } - ] - } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)" + ] + } + } + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SelectFields-Transform3_job2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "customProperties": { - "paths": "[]", - "transformation_ctx": "\"Transform3\"", - "transformType": "SelectFields", - "nodeId": "Transform3_job2" - }, - "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", - "name": "test-job-2:SelectFields-Transform3_job2", - "type": { - "string": "GLUE" - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)" - ], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)" - ] + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SelectFields-Transform3_job2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "paths": "[]", + "transformation_ctx": "\"Transform3\"", + "transformType": "SelectFields", + "nodeId": "Transform3_job2" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", + "name": "test-job-2:SelectFields-Transform3_job2", + "type": { + "string": "GLUE" } } - ] - } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)" + ], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)" + ] + } + } + ] } - }, - { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "connection_type": "s3", - "format": "json", - "connection_options": "{'path': 's3://test-glue-jsons/', 'partitionKeys': []}", - "transformation_ctx": "DataSink0" - }, - "tags": [] - } + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false } - ] - } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "connection_type": "s3", + "format": "json", + "connection_options": "{'path': 's3://test-glue-jsons/', 'partitionKeys': []}", + "transformation_ctx": "DataSink0" + }, + "tags": [] + } + } + ] } - }, - { - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(glue,test-job-1,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(glue,test-job-1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(glue,test-job-2,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(glue,test-job-2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform1_job1)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform1_job1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform5_job1)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform5_job1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Join-Transform3_job1)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Join-Transform3_job1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SelectFields-Transform3_job2)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SelectFields-Transform3_job2)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SplitFields-Transform0_job2)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SplitFields-Transform0_job2)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } - }, - { - "entityType": "tag", - "entityUrn": "urn:li:tag:baz:bob", - "changeType": "UPSERT", - "aspectName": "tagKey", - "aspect": { - "json": { - "name": "baz:bob" - } + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:baz:bob", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "baz:bob" } - }, - { - "entityType": "tag", - "entityUrn": "urn:li:tag:foo:bar", - "changeType": "UPSERT", - "aspectName": "tagKey", - "aspect": { - "json": { - "name": "foo:bar" - } + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:foo:bar", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "foo:bar" } } - ] \ No newline at end of file +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden_profiling.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden_profiling.json index 800d36a13d2dc..d9c6be11ed737 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden_profiling.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden_profiling.json @@ -16,7 +16,8 @@ "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "flights-database-profiling", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database-profiling" + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database-profiling", + "env": "PROD" } } }, @@ -266,8 +267,8 @@ "json": { "timestampMillis": 1586847600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "fieldProfiles": [ { diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden_table_column_lineage.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden_table_column_lineage.json index fd4109b0f93c9..718c520d420d8 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden_table_column_lineage.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden_table_column_lineage.json @@ -16,7 +16,8 @@ "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "flights-database-lineage", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database-lineage" + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database-lineage", + "env": "PROD" } } }, diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden_table_lineage.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden_table_lineage.json index 873776c5777bc..e2dd4cec97c2e 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden_table_lineage.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden_table_lineage.json @@ -16,7 +16,8 @@ "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "flights-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database" + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database", + "env": "PROD" } } }, @@ -69,7 +70,8 @@ "CreateTime": "June 01, 2021 at 14:55:02" }, "name": "test-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database" + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database", + "env": "PROD" } } }, @@ -122,7 +124,8 @@ "CreateTime": "June 01, 2021 at 14:55:13" }, "name": "empty-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database" + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database", + "env": "PROD" } } }, diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json index de82d26c57162..0b883062763f4 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json @@ -17,7 +17,8 @@ "CreateTime": "June 09, 2021 at 14:14:19" }, "name": "flights-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database" + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database", + "env": "PROD" } } }, @@ -72,7 +73,8 @@ "CreateTime": "June 01, 2021 at 14:55:02" }, "name": "test-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database" + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database", + "env": "PROD" } } }, @@ -127,7 +129,8 @@ "CreateTime": "June 01, 2021 at 14:55:13" }, "name": "empty-database", - "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database" + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database", + "env": "PROD" } } }, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java index 24598d7fdaadb..b7a04f2064d9b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java @@ -140,10 +140,15 @@ public SearchRequest getSearchRequest( opContext.getObjectMapper(), cac, queryWithDefaultFilters, - customQueryConfig)) + customQueryConfig, + input)) .orElse( SearchQueryBuilder.buildScoreFunctions( - opContext, customQueryConfig, List.of(entitySpec), queryWithDefaultFilters)); + opContext, + customQueryConfig, + List.of(entitySpec), + input, + queryWithDefaultFilters)); searchSourceBuilder.query(functionScoreQueryBuilder); ESUtils.buildSortOrder(searchSourceBuilder, null, List.of(entitySpec)); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandler.java index 0dbdf80860f7f..c4418401a844e 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/CustomizedQueryHandler.java @@ -119,16 +119,18 @@ private static BoolQueryBuilder toBoolQueryBuilder( public static FunctionScoreQueryBuilder functionScoreQueryBuilder( @Nonnull ObjectMapper objectMapper, @Nonnull QueryConfiguration customQueryConfiguration, - QueryBuilder queryBuilder) { + QueryBuilder queryBuilder, + String query) { return toFunctionScoreQueryBuilder( - objectMapper, queryBuilder, customQueryConfiguration.getFunctionScore()); + objectMapper, queryBuilder, customQueryConfiguration.getFunctionScore(), query); } public static Optional functionScoreQueryBuilder( @Nonnull ObjectMapper objectMapper, @Nonnull AutocompleteConfiguration customAutocompleteConfiguration, QueryBuilder queryBuilder, - @Nullable QueryConfiguration customQueryConfiguration) { + @Nullable QueryConfiguration customQueryConfiguration, + String query) { Optional result = Optional.empty(); @@ -143,14 +145,17 @@ public static Optional functionScoreQueryBuilder( result = Optional.of( toFunctionScoreQueryBuilder( - objectMapper, queryBuilder, customQueryConfiguration.getFunctionScore())); + objectMapper, queryBuilder, customQueryConfiguration.getFunctionScore(), query)); } else if (customAutocompleteConfiguration.getFunctionScore() != null && !customAutocompleteConfiguration.getFunctionScore().isEmpty()) { log.debug("Applying custom autocomplete function scores."); result = Optional.of( toFunctionScoreQueryBuilder( - objectMapper, queryBuilder, customAutocompleteConfiguration.getFunctionScore())); + objectMapper, + queryBuilder, + customAutocompleteConfiguration.getFunctionScore(), + query)); } return result; @@ -159,7 +164,8 @@ public static Optional functionScoreQueryBuilder( private static FunctionScoreQueryBuilder toFunctionScoreQueryBuilder( @Nonnull ObjectMapper objectMapper, @Nonnull QueryBuilder queryBuilder, - @Nonnull Map params) { + @Nonnull Map params, + String query) { try { HashMap body = new HashMap<>(params); if (!body.isEmpty()) { @@ -168,7 +174,12 @@ private static FunctionScoreQueryBuilder toFunctionScoreQueryBuilder( body.put("query", objectMapper.readValue(queryBuilder.toString(), Map.class)); - String jsonFragment = objectMapper.writeValueAsString(Map.of("function_score", body)); + String jsonFragment = + objectMapper + .writeValueAsString(Map.of("function_score", body)) + .replace("\"{{query_string}}\"", objectMapper.writeValueAsString(query)) + .replace( + "\"{{unquoted_query_string}}\"", objectMapper.writeValueAsString(unquote(query))); XContentParser parser = XContentType.JSON .xContent() diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java index 9ac460ed7e1e1..3e76d3600d6a6 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java @@ -78,7 +78,7 @@ public QueryBuilder buildQuery( final QueryBuilder queryBuilder = buildInternalQuery(opContext, customQueryConfig, entitySpecs, query, fulltext); - return buildScoreFunctions(opContext, customQueryConfig, entitySpecs, queryBuilder); + return buildScoreFunctions(opContext, customQueryConfig, entitySpecs, query, queryBuilder); } /** @@ -485,12 +485,13 @@ static FunctionScoreQueryBuilder buildScoreFunctions( @Nonnull OperationContext opContext, @Nullable QueryConfiguration customQueryConfig, @Nonnull List entitySpecs, + String query, @Nonnull QueryBuilder queryBuilder) { if (customQueryConfig != null) { // Prefer configuration function scoring over annotation scoring return CustomizedQueryHandler.functionScoreQueryBuilder( - opContext.getObjectMapper(), customQueryConfig, queryBuilder); + opContext.getObjectMapper(), customQueryConfig, queryBuilder, query); } else { return QueryBuilders.functionScoreQuery( queryBuilder, buildAnnotationScoreFunctions(entitySpecs)) diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java index 4e4c8acf300e4..a16e497ff548f 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java @@ -179,7 +179,7 @@ public void functionScoreQueryBuilderTest() { */ FunctionScoreQueryBuilder selectStarTest = CustomizedQueryHandler.functionScoreQueryBuilder( - new ObjectMapper(), test.lookupQueryConfig("*").get(), inputQuery); + new ObjectMapper(), test.lookupQueryConfig("*").get(), inputQuery, "*"); FunctionScoreQueryBuilder.FilterFunctionBuilder[] expectedSelectStarScoreFunctions = { new FunctionScoreQueryBuilder.FilterFunctionBuilder( @@ -203,7 +203,7 @@ public void functionScoreQueryBuilderTest() { */ FunctionScoreQueryBuilder defaultTest = CustomizedQueryHandler.functionScoreQueryBuilder( - new ObjectMapper(), test.lookupQueryConfig("foobar").get(), inputQuery); + new ObjectMapper(), test.lookupQueryConfig("foobar").get(), inputQuery, "foobar"); FunctionScoreQueryBuilder.FilterFunctionBuilder[] expectedDefaultScoreFunctions = { new FunctionScoreQueryBuilder.FilterFunctionBuilder( diff --git a/metadata-models/src/main/pegasus/com/linkedin/container/ContainerProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/container/ContainerProperties.pdl index 526878cbe60d3..ad91f4b85d35c 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/container/ContainerProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/container/ContainerProperties.pdl @@ -3,6 +3,7 @@ namespace com.linkedin.container import com.linkedin.common.CustomProperties import com.linkedin.common.ExternalReference import com.linkedin.common.TimeStamp +import com.linkedin.common.FabricType /** * Information about a Asset Container as received from a 3rd party source system @@ -41,6 +42,17 @@ record ContainerProperties includes CustomProperties, ExternalReference { } description: optional string + /** + * Environment for this flow + */ + @Searchable = { + "fieldType": "KEYWORD", + "addToFilters": true, + "filterNameOverride": "Environment", + "queryByDefault": false + } + env: optional FabricType + /** * A timestamp documenting when the asset was created in the source Data Platform (not on DataHub) */ diff --git a/smoke-test/tests/tokens/revokable_access_token_test.py b/smoke-test/tests/tokens/revokable_access_token_test.py index e2fabce8ac4e8..7447cba60b9bb 100644 --- a/smoke-test/tests/tokens/revokable_access_token_test.py +++ b/smoke-test/tests/tokens/revokable_access_token_test.py @@ -24,10 +24,11 @@ def wait_for_healthchecks(): yield +@pytest.fixture(autouse=True) @pytest.mark.dependency() def test_healthchecks(wait_for_healthchecks): # Call to wait_for_healthchecks fixture will do the actual functionality. - pass + wait_for_writes_to_sync() @pytest.mark.dependency(depends=["test_healthchecks"])