From 8d90c21f804306b2e0f1aa9aac694c7b057b6813 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Mon, 22 Apr 2024 10:07:52 -0500 Subject: [PATCH 1/4] =?UTF-8?q?feat(schema-registry):=20exclude=20schema?= =?UTF-8?q?=20reg=20onboot=20check=20from=20schema=20re=E2=80=A6=20(#10349?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../metadata/boot/OnBootApplicationListener.java | 9 +++++++++ .../java/com/linkedin/gms/WebApplicationInitializer.java | 8 +++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java index 301e3f70571fb8..ced315ba6e65fd 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java @@ -26,6 +26,9 @@ @Slf4j @Component public class OnBootApplicationListener { + + public static final String SCHEMA_REGISTRY_SERVLET_NAME = "dispatcher-schema-registry"; + private static final Set ACCEPTED_HTTP_CODES = Set.of( HttpStatus.SC_OK, @@ -58,6 +61,12 @@ public class OnBootApplicationListener { @EventListener(ContextRefreshedEvent.class) public void onApplicationEvent(@Nonnull ContextRefreshedEvent event) { + + if (SCHEMA_REGISTRY_SERVLET_NAME.equals(event.getApplicationContext().getId())) { + log.info("Loading servlet {} without interruption.", SCHEMA_REGISTRY_SERVLET_NAME); + return; + } + log.warn( "OnBootApplicationListener context refreshed! {} event: {}", ROOT_WEB_APPLICATION_CONTEXT_ID.equals(event.getApplicationContext().getId()), diff --git a/metadata-service/war/src/main/java/com/linkedin/gms/WebApplicationInitializer.java b/metadata-service/war/src/main/java/com/linkedin/gms/WebApplicationInitializer.java index c2a7e99c949e45..fd32610be35321 100644 --- a/metadata-service/war/src/main/java/com/linkedin/gms/WebApplicationInitializer.java +++ b/metadata-service/war/src/main/java/com/linkedin/gms/WebApplicationInitializer.java @@ -1,5 +1,7 @@ package com.linkedin.gms; +import static com.linkedin.metadata.boot.OnBootApplicationListener.SCHEMA_REGISTRY_SERVLET_NAME; + import com.datahub.auth.authentication.filter.AuthenticationFilter; import com.datahub.gms.servlet.Config; import com.datahub.gms.servlet.ConfigSearchExport; @@ -35,6 +37,7 @@ public void onStartup(ServletContext container) { "contextInitializerClasses", "com.linkedin.gms.SpringApplicationInitializer"); // Auth filter + List servletNames = new ArrayList<>(); // Independent dispatcher schemaRegistryServlet(container); @@ -43,8 +46,6 @@ public void onStartup(ServletContext container) { healthCheckServlet(container); configServlet(container); - List servletNames = new ArrayList<>(); - // Restli non-Dispatcher servletNames.add(restliServlet(rootContext, container)); @@ -69,11 +70,12 @@ public void onStartup(ServletContext container) { */ private void schemaRegistryServlet(ServletContext container) { AnnotationConfigWebApplicationContext webContext = new AnnotationConfigWebApplicationContext(); + webContext.setId(SCHEMA_REGISTRY_SERVLET_NAME); webContext.register(SchemaRegistryServletConfig.class); DispatcherServlet dispatcherServlet = new DispatcherServlet(webContext); ServletRegistration.Dynamic registration = - container.addServlet("dispatcher-schema-registry", dispatcherServlet); + container.addServlet(SCHEMA_REGISTRY_SERVLET_NAME, dispatcherServlet); registration.addMapping("/schema-registry/*"); registration.setLoadOnStartup(1); registration.setAsyncSupported(true); From 725c85815bfc83e2accf1bbb5fa2c9be637dde88 Mon Sep 17 00:00:00 2001 From: Sergio Date: Mon, 22 Apr 2024 18:12:41 +0200 Subject: [PATCH 2/4] fix(ingest/starburst): parse create_time datetime format (#10345) --- .../src/datahub/ingestion/source/usage/starburst_trino_usage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py index e71f640c142561..10cf981abce562 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py @@ -191,6 +191,8 @@ def _convert_str_to_datetime(self, v): if isinstance(v, str): isodate = parser.parse(v) # compatible with Python 3.6+ return isodate + if isinstance(v, datetime): + return v def _get_joined_access_event(self, events): joined_access_events = [] From 934ab03d16dc52f992a807a2002e9949cc6f95fa Mon Sep 17 00:00:00 2001 From: Shubham Jagtap <132359390+shubhamjagtap639@users.noreply.github.com> Date: Tue, 23 Apr 2024 04:49:01 +0530 Subject: [PATCH 3/4] test(ingestion/sigma): Add integration test cases (#10356) --- .../datahub/ingestion/source/sigma/sigma.py | 2 +- .../golden_test_platform_instance_ingest.json | 1972 +++++++++++++++++ .../sigma/golden_test_sigma_ingest.json | 1859 ++++++++++++++++ .../tests/integration/sigma/test_sigma.py | 493 +++++ 4 files changed, 4325 insertions(+), 1 deletion(-) create mode 100644 metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json create mode 100644 metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json create mode 100644 metadata-ingestion/tests/integration/sigma/test_sigma.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py index 46e92e8533a21e..97da198aaa435a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py @@ -253,7 +253,7 @@ def _gen_entity_browsepath_aspect( urn = builder.make_dataplatform_instance_urn( self.platform, self.config.platform_instance ) - entries = [BrowsePathEntryClass(id=urn, urn=urn)] + entries + entries = [BrowsePathEntryClass(id=urn, urn=urn)] + entries return MetadataChangeProposalWrapper( entityUrn=entity_urn, aspect=BrowsePathsV2Class(entries), diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json new file mode 100644 index 00000000000000..a81e6e927ef6ac --- /dev/null +++ b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json @@ -0,0 +1,1972 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713794496006, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + } + }, + "systemMetadata": { + "lastObserved": 1713794496008, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "path": "Acryl Data" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/b/49HFLTr6xytgrPly3PFsNC", + "name": "PETS", + "qualifiedName": "PETS", + "description": "", + "created": { + "time": 1713188592664 + }, + "lastModified": { + "time": 1713188592664 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1713794496007, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + } + }, + "systemMetadata": { + "lastObserved": 1713794496008, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496010, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496009, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1713794496009, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713794496011, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + } + }, + "systemMetadata": { + "lastObserved": 1713794496012, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "path": "Acryl Data/New Folder" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/b/5LqGLu14qUnqh3cN6wRJBd", + "name": "PET_PROFILES_JOINED_DYNAMIC", + "qualifiedName": "PET_PROFILES_JOINED_DYNAMIC", + "description": "", + "created": { + "time": 1713189068019 + }, + "lastModified": { + "time": 1713189068019 + }, + "tags": [ + "Deprecated" + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496012, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1713794496201, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + } + }, + "systemMetadata": { + "lastObserved": 1713794496013, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496200, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "New Folder" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496015, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + } + }, + "systemMetadata": { + "lastObserved": 1713794496200, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496014, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Deprecated" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496015, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1713794496014, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713794496199, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "sigma", + "instance": "cloud_instance", + "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b" + }, + "name": "Acryl Data", + "created": { + "time": 1710232264826 + }, + "lastModified": { + "time": 1710232264826 + } + } + }, + "systemMetadata": { + "lastObserved": 1713794496199, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713794496053, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496202, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + } + }, + "systemMetadata": { + "lastObserved": 1713794496055, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496055, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "levelTable", + "type": "table" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=kH0MeihtGs&:fullScreen=true", + "title": "ADOPTIONS", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496054, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496058, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1713794496019, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713794496021, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "ElementsCount": "2" + }, + "title": "Page 1", + "description": "", + "charts": [ + "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", + "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1713794496022, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "sigma", + "instance": "cloud_instance", + "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", + "path": "Acryl Data", + "latestVersion": "2" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", + "name": "Acryl Workbook", + "created": { + "time": 1713188691477 + }, + "lastModified": { + "time": 1713189117302 + } + } + }, + "systemMetadata": { + "lastObserved": 1713794496016, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + } + }, + "systemMetadata": { + "lastObserved": 1713794496023, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496018, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713794496017, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + } + }, + "systemMetadata": { + "lastObserved": 1713794496023, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496024, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Warning" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496019, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + } + }, + "systemMetadata": { + "lastObserved": 1713794496018, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713794496114, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496020, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + } + }, + "systemMetadata": { + "lastObserved": 1713794496116, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Count of Profile Id)", + "schemaField": { + "fieldPath": "Count of Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496117, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "bar", + "type": "visualization" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=Ml9C5ezT5W&:fullScreen=true", + "title": "Count of Profile Id by Status", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496115, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496119, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713794496124, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "ElementsCount": "1" + }, + "title": "Page 2", + "description": "", + "charts": [ + "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1713794496125, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + } + }, + "systemMetadata": { + "lastObserved": 1713794496020, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + } + }, + "systemMetadata": { + "lastObserved": 1713794496125, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "levelTable", + "type": "table" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=tQJu5N1l81&:fullScreen=true", + "title": "PETS ADOPTIONS JOIN", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496188, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713794496188, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + } + }, + "systemMetadata": { + "lastObserved": 1713794496126, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496126, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + } + }, + "systemMetadata": { + "lastObserved": 1713794496189, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Count of Profile Id)", + "schemaField": { + "fieldPath": "Count of Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496120, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Pk %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Pk (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Status %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Status (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Created At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Created At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Updated At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Updated At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496194, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496194, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Pk %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Pk (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Status %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Status (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Created At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Created At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.tQJu5N1l81),Updated At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Updated At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496190, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Warning", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Warning" + } + }, + "systemMetadata": { + "lastObserved": 1713794496203, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Deprecated", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Deprecated" + } + }, + "systemMetadata": { + "lastObserved": 1713794496203, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.pets,DEV)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713794496202, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json new file mode 100644 index 00000000000000..50865ab0ff72e3 --- /dev/null +++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json @@ -0,0 +1,1859 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713795619218, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "path": "Acryl Data" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/b/49HFLTr6xytgrPly3PFsNC", + "name": "PETS", + "qualifiedName": "PETS", + "description": "", + "created": { + "time": 1713188592664 + }, + "lastModified": { + "time": 1713188592664 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1713795619219, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1713795619461, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma" + } + }, + "systemMetadata": { + "lastObserved": 1713795619459, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1713795619220, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + }, + "systemMetadata": { + "lastObserved": 1713795619220, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "sigma", + "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b" + }, + "name": "Acryl Data", + "created": { + "time": 1710232264826 + }, + "lastModified": { + "time": 1710232264826 + } + } + }, + "systemMetadata": { + "lastObserved": 1713795619458, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619221, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619460, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713795619459, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619221, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1713795619460, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713795619222, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "sigma", + "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", + "path": "Acryl Data", + "latestVersion": "2" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", + "name": "Acryl Workbook", + "created": { + "time": 1713188691477 + }, + "lastModified": { + "time": 1713189117302 + } + } + }, + "systemMetadata": { + "lastObserved": 1713795619227, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713795619228, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "path": "Acryl Data/New Folder" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/b/5LqGLu14qUnqh3cN6wRJBd", + "name": "PET_PROFILES_JOINED_DYNAMIC", + "qualifiedName": "PET_PROFILES_JOINED_DYNAMIC", + "description": "", + "created": { + "time": 1713189068019 + }, + "lastModified": { + "time": 1713189068019 + }, + "tags": [ + "Deprecated" + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619223, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "New Folder" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619226, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1713795619229, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619229, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma" + } + }, + "systemMetadata": { + "lastObserved": 1713795619228, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + }, + "systemMetadata": { + "lastObserved": 1713795619224, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713795619232, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "ElementsCount": "2" + }, + "title": "Page 1", + "description": "", + "charts": [ + "urn:li:chart:(sigma,kH0MeihtGs)", + "urn:li:chart:(sigma,Ml9C5ezT5W)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1713795619233, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1713795619224, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Warning" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619230, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1713795619234, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Deprecated" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619226, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619234, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + }, + "systemMetadata": { + "lastObserved": 1713795619231, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619225, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619375, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619231, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1713795619373, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Count of Profile Id)", + "schemaField": { + "fieldPath": "Count of Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619373, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713795619371, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "bar", + "type": "visualization" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=Ml9C5ezT5W&:fullScreen=true", + "title": "Count of Profile Id by Status", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619372, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713795619381, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "ElementsCount": "1" + }, + "title": "Page 2", + "description": "", + "charts": [ + "urn:li:chart:(sigma,tQJu5N1l81)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1713795619381, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619270, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,Ml9C5ezT5W),Count of Profile Id)", + "schemaField": { + "fieldPath": "Count of Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619376, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713795619447, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "levelTable", + "type": "table" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=tQJu5N1l81&:fullScreen=true", + "title": "PETS ADOPTIONS JOIN", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.adoptions,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619447, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1713795619382, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1713795619267, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + }, + "systemMetadata": { + "lastObserved": 1713795619449, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pk %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Pk (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Status %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Status (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Created At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Created At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Updated At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Updated At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619449, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619383, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,kH0MeihtGs),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619267, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619453, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1713795619265, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "levelTable", + "type": "table" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=kH0MeihtGs&:fullScreen=true", + "title": "ADOPTIONS", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.adoptions,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619266, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.pets,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619462, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Profile Id)", + "schemaField": { + "fieldPath": "Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pk %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Pk (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Status %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Status (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Created At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Created At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,tQJu5N1l81),Updated At %28ADOPTIONS%29)", + "schemaField": { + "fieldPath": "Updated At (ADOPTIONS)", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1713795619454, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Deprecated", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Deprecated" + } + }, + "systemMetadata": { + "lastObserved": 1713795619462, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Warning", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Warning" + } + }, + "systemMetadata": { + "lastObserved": 1713795619463, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sigma/test_sigma.py b/metadata-ingestion/tests/integration/sigma/test_sigma.py new file mode 100644 index 00000000000000..c9a845e59ddf25 --- /dev/null +++ b/metadata-ingestion/tests/integration/sigma/test_sigma.py @@ -0,0 +1,493 @@ +from typing import Any, Dict + +import pytest + +from datahub.ingestion.run.pipeline import Pipeline +from tests.test_helpers import mce_helpers + + +def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: + api_vs_response: Dict[str, Dict] = { + "https://aws-api.sigmacomputing.com/v2/auth/token": { + "method": "POST", + "status_code": 200, + "json": { + "access_token": "717de8281754fe8e302b1ee69f1c9553faf0331cabd8712f459c", + "token_type": "bearer", + "expires_in": 3599, + }, + }, + "https://aws-api.sigmacomputing.com/v2/files": { + "method": "GET", + "status_code": 200, + "json": { + "entries": [ + { + "id": "8891fd40-5470-4ff2-a74f-6e61ee44d3fc", + "urlId": "49HFLTr6xytgrPly3PFsNC", + "name": "PETS", + "type": "dataset", + "parentId": "3ee61405-3be2-4000-ba72-60d36757b95b", + "parentUrlId": "1UGFyEQCHqwPfQoAec3xJ9", + "permission": "edit", + "path": "Acryl Data", + "badge": None, + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-04-15T13:43:12.664Z", + "updatedAt": "2024-04-15T13:43:12.664Z", + "isArchived": False, + }, + { + "id": "bd6b86e8-cd4a-4b25-ab65-f258c2a68a8f", + "urlId": "5LqGLu14qUnqh3cN6wRJBd", + "name": "PET_PROFILES_JOINED_DYNAMIC", + "type": "dataset", + "parentId": "1b47afdb-db4e-4a2c-9fa4-fc1332f4a097", + "parentUrlId": "Ptyl1jrKEO18RDX9y1d4P", + "permission": "edit", + "path": "Acryl Data/New Folder", + "badge": "Deprecated", + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-04-15T13:51:08.019Z", + "updatedAt": "2024-04-15T13:51:08.019Z", + "isArchived": False, + }, + { + "id": "1b47afdb-db4e-4a2c-9fa4-fc1332f4a097", + "urlId": "Ptyl1jrKEO18RDX9y1d4P", + "name": "New Folder", + "type": "folder", + "parentId": "3ee61405-3be2-4000-ba72-60d36757b95b", + "parentUrlId": "1UGFyEQCHqwPfQoAec3xJ9", + "permission": "edit", + "path": "Acryl Data", + "badge": None, + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-04-15T13:35:39.512Z", + "updatedAt": "2024-04-15T13:35:39.512Z", + "isArchived": False, + }, + { + "id": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", + "urlId": "4JRFW1HThPI1K3YTjouXI7", + "name": "Acryl Workbook", + "type": "workbook", + "parentId": "3ee61405-3be2-4000-ba72-60d36757b95b", + "parentUrlId": "1UGFyEQCHqwPfQoAec3xJ9", + "permission": "edit", + "path": "Acryl Data", + "badge": "Warning", + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-04-15T13:44:51.477Z", + "updatedAt": "2024-04-15T13:51:57.302Z", + "isArchived": False, + }, + ], + "total": 4, + "nextPage": None, + }, + }, + "https://aws-api.sigmacomputing.com/v2/workspaces/3ee61405-3be2-4000-ba72-60d36757b95b": { + "method": "GET", + "status_code": 200, + "json": { + "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b", + "name": "Acryl Data", + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-03-12T08:31:04.826Z", + "updatedAt": "2024-03-12T08:31:04.826Z", + }, + }, + "https://aws-api.sigmacomputing.com/v2/datasets/8891fd40-5470-4ff2-a74f-6e61ee44d3fc": { + "method": "GET", + "status_code": 200, + "json": { + "datasetId": "8891fd40-5470-4ff2-a74f-6e61ee44d3fc", + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-04-15T13:43:12.664Z", + "updatedAt": "2024-04-15T13:43:12.664Z", + "name": "PETS", + "description": "", + "url": "https://app.sigmacomputing.com/acryldata/b/49HFLTr6xytgrPly3PFsNC", + "isArchived": False, + }, + }, + "https://aws-api.sigmacomputing.com/v2/files/1b47afdb-db4e-4a2c-9fa4-fc1332f4a097": { + "method": "GET", + "status_code": 200, + "json": { + "id": "1b47afdb-db4e-4a2c-9fa4-fc1332f4a097", + "urlId": "Ptyl1jrKEO18RDX9y1d4P", + "name": "New Folder", + "type": "folder", + "parentId": "3ee61405-3be2-4000-ba72-60d36757b95b", + "parentUrlId": "1UGFyEQCHqwPfQoAec3xJ9", + "permission": "edit", + "path": "Acryl Data", + "badge": None, + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-04-15T13:35:39.512Z", + "updatedAt": "2024-04-15T13:35:39.512Z", + "isArchived": False, + }, + }, + "https://aws-api.sigmacomputing.com/v2/datasets/bd6b86e8-cd4a-4b25-ab65-f258c2a68a8f": { + "method": "GET", + "status_code": 200, + "json": { + "datasetId": "bd6b86e8-cd4a-4b25-ab65-f258c2a68a8f", + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-04-15T13:51:08.019Z", + "updatedAt": "2024-04-15T13:51:08.019Z", + "name": "PET_PROFILES_JOINED_DYNAMIC", + "description": "", + "url": "https://app.sigmacomputing.com/acryldata/b/5LqGLu14qUnqh3cN6wRJBd", + "isArchived": False, + "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b", + "path": "Acryl Data/New Folder", + }, + }, + "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b": { + "method": "GET", + "status_code": 200, + "json": { + "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", + "workbookUrlId": "4JRFW1HThPI1K3YTjouXI7", + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2024-04-15T13:44:51.477Z", + "updatedAt": "2024-04-15T13:51:57.302Z", + "name": "Acryl Workbook", + "url": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", + "path": "Acryl Data", + "latestVersion": 2, + "isArchived": False, + "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b", + }, + }, + "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/pages": { + "method": "GET", + "status_code": 200, + "json": { + "entries": [ + {"pageId": "OSnGLBzL1i", "name": "Page 1"}, + {"pageId": "DFSieiAcgo", "name": "Page 2"}, + ], + "total": 2, + "nextPage": None, + }, + }, + "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/pages/OSnGLBzL1i/elements": { + "method": "GET", + "status_code": 200, + "json": { + "entries": [ + { + "elementId": "kH0MeihtGs", + "type": "table", + "name": "ADOPTIONS", + "columns": [ + "Pk", + "Pet Fk", + "Human Fk", + "Status", + "Created At", + "Updated At", + ], + "vizualizationType": "levelTable", + }, + { + "elementId": "Ml9C5ezT5W", + "type": "visualization", + "name": "Count of Profile Id by Status", + "columns": [ + "Pk", + "Profile Id", + "Status", + "Created At", + "Updated At", + "Count of Profile Id", + ], + "vizualizationType": "bar", + }, + ], + "total": 2, + "nextPage": None, + }, + }, + "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/lineage/elements/kH0MeihtGs": { + "method": "GET", + "status_code": 200, + "json": { + "dependencies": { + "mnJ7_k2sbt": { + "nodeId": "mnJ7_k2sbt", + "type": "sheet", + "name": "ADOPTIONS", + "elementId": "kH0MeihtGs", + }, + "inode-2Fby2MBLPM5jUMfBB15On1": { + "nodeId": "inode-2Fby2MBLPM5jUMfBB15On1", + "type": "table", + "name": "ADOPTIONS", + }, + }, + "edges": [ + { + "source": "inode-2Fby2MBLPM5jUMfBB15On1", + "target": "mnJ7_k2sbt", + "type": "source", + } + ], + }, + }, + "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/elements/kH0MeihtGs/query": { + "method": "GET", + "status_code": 200, + "json": { + "elementId": "kH0MeihtGs", + "name": "ADOPTIONS", + "sql": 'select PK "Pk", PET_FK "Pet Fk", HUMAN_FK "Human Fk", STATUS "Status", CAST_TIMESTAMP_TO_DATETIME_7 "Created At", CAST_TIMESTAMP_TO_DATETIME_8 "Updated At" from (select PK, PET_FK, HUMAN_FK, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_8 from (select * from LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS ADOPTIONS limit 1000) Q1) Q2 limit 1000\n\n-- Sigma Σ {"request-id":"3d4bf15e-6a17-4967-ad2a-213341233a04","email":"Shubham.Jagtap@gslab.com"}', + }, + }, + "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/lineage/elements/Ml9C5ezT5W": { + "method": "GET", + "status_code": 200, + "json": { + "dependencies": { + "qrL7BEq8LR": { + "nodeId": "qrL7BEq8LR", + "type": "sheet", + "name": "Count of Profile Id by Status", + "elementId": "Ml9C5ezT5W", + }, + "inode-49HFLTr6xytgrPly3PFsNC": { + "nodeId": "inode-49HFLTr6xytgrPly3PFsNC", + "type": "dataset", + "name": "PETS", + }, + }, + "edges": [ + { + "source": "inode-49HFLTr6xytgrPly3PFsNC", + "target": "qrL7BEq8LR", + "type": "source", + } + ], + }, + }, + "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/elements/Ml9C5ezT5W/query": { + "method": "GET", + "status_code": 200, + "json": { + "elementId": "Ml9C5ezT5W", + "name": "Count of Profile Id by Status", + "sql": 'with Q1 as (select PK, PROFILE_ID, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_6, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) select STATUS_10 "Status", COUNT_23 "Count of Profile Id", PK_8 "Pk", PROFILE_ID_9 "Profile Id", CAST_TIMESTAMP_TO_DATETIME_8 "Created At", CAST_TIMESTAMP_TO_DATETIME_9 "Updated At" from (select Q3.PK_8 PK_8, Q3.PROFILE_ID_9 PROFILE_ID_9, Q3.STATUS_10 STATUS_10, Q3.CAST_TIMESTAMP_TO_DATETIME_8 CAST_TIMESTAMP_TO_DATETIME_8, Q3.CAST_TIMESTAMP_TO_DATETIME_9 CAST_TIMESTAMP_TO_DATETIME_9, Q6.COUNT_23 COUNT_23, Q6.STATUS_11 STATUS_11 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CAST_TIMESTAMP_TO_DATETIME_6 CAST_TIMESTAMP_TO_DATETIME_8, CAST_TIMESTAMP_TO_DATETIME_7 CAST_TIMESTAMP_TO_DATETIME_9 from Q1 Q2 order by STATUS_10 asc limit 1000) Q3 left join (select count(PROFILE_ID_9) COUNT_23, STATUS_10 STATUS_11 from (select PROFILE_ID PROFILE_ID_9, STATUS STATUS_10 from Q1 Q4) Q5 group by STATUS_10) Q6 on equal_null(Q3.STATUS_10, Q6.STATUS_11)) Q8 order by STATUS_10 asc limit 1000\n\n-- Sigma Σ {"request-id":"988dd6b5-0678-4421-ae14-21594c0ee97a","email":"Shubham.Jagtap@gslab.com"}', + }, + }, + "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/pages/DFSieiAcgo/elements": { + "method": "GET", + "status_code": 200, + "json": { + "entries": [ + { + "elementId": "tQJu5N1l81", + "type": "table", + "name": "PETS ADOPTIONS JOIN", + "columns": [ + "Pk", + "Profile Id", + "Status", + "Created At", + "Updated At", + "Pk (ADOPTIONS)", + "Pet Fk", + "Human Fk", + "Status (ADOPTIONS)", + "Created At (ADOPTIONS)", + "Updated At (ADOPTIONS)", + ], + "vizualizationType": "levelTable", + } + ], + "total": 1, + "nextPage": None, + }, + }, + "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/lineage/elements/tQJu5N1l81": { + "method": "GET", + "status_code": 200, + "json": { + "dependencies": { + "QGgIlk8PQk": { + "nodeId": "QGgIlk8PQk", + "type": "sheet", + "name": "PETS ADOPTIONS JOIN", + "elementId": "tQJu5N1l81", + }, + "2zTHG9wyvZ": {"nodeId": "2zTHG9wyvZ", "type": "join"}, + "inode-49HFLTr6xytgrPly3PFsNC": { + "nodeId": "inode-49HFLTr6xytgrPly3PFsNC", + "type": "dataset", + "name": "PETS", + }, + "inode-2Fby2MBLPM5jUMfBB15On1": { + "nodeId": "inode-2Fby2MBLPM5jUMfBB15On1", + "type": "table", + "name": "ADOPTIONS", + }, + }, + "edges": [ + {"source": "2zTHG9wyvZ", "target": "QGgIlk8PQk", "type": "source"}, + { + "source": "inode-49HFLTr6xytgrPly3PFsNC", + "target": "2zTHG9wyvZ", + "type": "source", + }, + { + "source": "inode-2Fby2MBLPM5jUMfBB15On1", + "target": "2zTHG9wyvZ", + "type": "source", + }, + ], + }, + }, + "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/elements/tQJu5N1l81/query": { + "method": "GET", + "status_code": 200, + "json": { + "elementId": "tQJu5N1l81", + "name": "PETS ADOPTIONS JOIN", + "sql": 'select PK_8 "Pk", PROFILE_ID_9 "Profile Id", STATUS_10 "Status", CAST_TIMESTAMP_TO_DATETIME_11 "Created At", CAST_TIMESTAMP_TO_DATETIME_12 "Updated At", PK_13 "Pk (ADOPTIONS)", PET_FK_14 "Pet Fk", HUMAN_FK_15 "Human Fk", STATUS_16 "Status (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_19 "Created At (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_20 "Updated At (ADOPTIONS)" from (select PK_8, PROFILE_ID_9, STATUS_10, CAST_TIMESTAMP_TO_DATETIME_11, CAST_TIMESTAMP_TO_DATETIME_12, PK_13, PET_FK_14, HUMAN_FK_15, STATUS_16, CREATED_AT_17::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_19, UPDATED_AT_18::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_20 from (select Q1.PK_8 PK_8, Q1.PROFILE_ID_9 PROFILE_ID_9, Q1.STATUS_10 STATUS_10, Q1.CAST_TIMESTAMP_TO_DATETIME_11 CAST_TIMESTAMP_TO_DATETIME_11, Q1.CAST_TIMESTAMP_TO_DATETIME_12 CAST_TIMESTAMP_TO_DATETIME_12, Q2.PK PK_13, Q2.PET_FK PET_FK_14, Q2.HUMAN_FK HUMAN_FK_15, Q2.STATUS STATUS_16, Q2.CREATED_AT CREATED_AT_17, Q2.UPDATED_AT UPDATED_AT_18 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_11, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_12 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) Q1 inner join LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS Q2 on (Q1.PK_8 = Q2.PET_FK) limit 1000) Q4) Q5 limit 1000\n\n-- Sigma Σ {"request-id":"f5a997ef-b80c-47f1-b32e-9cd0f50cd491","email":"Shubham.Jagtap@gslab.com"}', + }, + }, + "https://aws-api.sigmacomputing.com/v2/members/CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1": { + "method": "GET", + "status_code": 200, + "json": { + "organizationId": "b94da709-176c-4242-bea6-6760f34c9228", + "memberId": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "memberType": "admin", + "firstName": "Shubham", + "lastName": "Jagtap", + "email": "Shubham.Jagtap@gslab.com", + "profileImgUrl": None, + "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", + "createdAt": "2023-11-28T10:59:20.957Z", + "updatedAt": "2024-03-12T21:21:17.996Z", + "homeFolderId": "9bb94df1-e8af-49eb-9c37-2bd40b0efb2e", + "userKind": "internal", + "isArchived": False, + "isInactive": False, + }, + }, + } + + api_vs_response.update(override_data) + + for url in api_vs_response.keys(): + request_mock.register_uri( + api_vs_response[url]["method"], + url, + json=api_vs_response[url]["json"], + status_code=api_vs_response[url]["status_code"], + ) + + +@pytest.mark.integration +def test_sigma_ingest(pytestconfig, tmp_path, requests_mock): + + test_resources_dir = pytestconfig.rootpath / "tests/integration/sigma" + + register_mock_api(request_mock=requests_mock) + + output_path: str = f"{tmp_path}/sigma_ingest_mces.json" + + pipeline = Pipeline.create( + { + "run_id": "sigma-test", + "source": { + "type": "sigma", + "config": { + "client_id": "CLIENTID", + "client_secret": "CLIENTSECRET", + "chart_sources_platform_mapping": { + "Acryl Data/Acryl Workbook": { + "data_source_platform": "snowflake" + }, + }, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": output_path, + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_sigma_ingest.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_path, + golden_path=f"{test_resources_dir}/{golden_file}", + ) + + +@pytest.mark.integration +def test_platform_instance_ingest(pytestconfig, tmp_path, requests_mock): + + test_resources_dir = pytestconfig.rootpath / "tests/integration/sigma" + + register_mock_api(request_mock=requests_mock) + + output_path: str = f"{tmp_path}/sigma_platform_instace_ingest_mces.json" + + pipeline = Pipeline.create( + { + "run_id": "sigma-test", + "source": { + "type": "sigma", + "config": { + "client_id": "CLIENTID", + "client_secret": "CLIENTSECRET", + "platform_instance": "cloud_instance", + "chart_sources_platform_mapping": { + "Acryl Data/Acryl Workbook": { + "data_source_platform": "snowflake", + "platform_instance": "dev_instance", + "env": "DEV", + }, + }, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": output_path, + }, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_platform_instance_ingest.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_path, + golden_path=f"{test_resources_dir}/{golden_file}", + ) From b9a34fe90b93ce547a5eeb09f6af6ecffebab322 Mon Sep 17 00:00:00 2001 From: dushayntAW <158567391+dushayntAW@users.noreply.github.com> Date: Tue, 23 Apr 2024 17:05:55 +0530 Subject: [PATCH 4/4] fix(ingest/salesforce): escape markdown char for multiline description (#10351) --- .../datahub/ingestion/source/salesforce.py | 27 ++++++++++--------- .../account_fields_soql_response.json | 4 ++- .../salesforce/salesforce_mces_golden.json | 2 +- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py index d19bc42d2111f3..29604115744307 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py +++ b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py @@ -537,11 +537,21 @@ def get_profile_workunit( # Here field description is created from label, description and inlineHelpText def _get_field_description(self, field: dict, customField: dict) -> str: - desc = field["Label"] - if field.get("FieldDefinition", {}).get("Description"): - desc = "{0}\n\n{1}".format(desc, field["FieldDefinition"]["Description"]) - if field.get("InlineHelpText"): - desc = "{0}\n\n{1}".format(desc, field["InlineHelpText"]) + desc = ( + "\\" + field["Label"] if field["Label"].startswith("#") else field["Label"] + ) + + for key in ["FieldDefinition", "InlineHelpText"]: + text: Optional[str] = "" + if isinstance(field.get(key), dict): + text = field[key].get("Description") + else: + text = field.get(key) + + if text: + prefix = "\\" if text.startswith("#") else "" + desc += f"\n\n{prefix}{text}" + return desc # Here jsonProps is used to add additional salesforce field level properties. @@ -575,13 +585,6 @@ def _get_schema_field( description = self._get_field_description(field, customField) - # escaping string starting with `#` - description = ( - "\\" + description - if description and description.startswith("#") - else description - ) - schemaField = SchemaFieldClass( fieldPath=fieldPath, type=SchemaFieldDataTypeClass(type=TypeClass()), # type:ignore diff --git a/metadata-ingestion/tests/integration/salesforce/mock_files/account_fields_soql_response.json b/metadata-ingestion/tests/integration/salesforce/mock_files/account_fields_soql_response.json index 138c5743235888..d00f4b1832418f 100644 --- a/metadata-ingestion/tests/integration/salesforce/mock_files/account_fields_soql_response.json +++ b/metadata-ingestion/tests/integration/salesforce/mock_files/account_fields_soql_response.json @@ -2517,6 +2517,7 @@ "QualifiedApiName": "Unique_Account", "DeveloperName": "Unique_Account", "Label": "# Unique_Account", + "InlineHelpText": "# Help Text", "FieldDefinition": { "attributes": { "type": "FieldDefinition", @@ -2527,7 +2528,8 @@ "LastModifiedBy": null, "IsIndexed": false, "ComplianceGroup": null, - "SecurityClassification": null + "SecurityClassification": null, + "Description": "This is the # description" }, "DataType": "string", "Precision": 0, diff --git a/metadata-ingestion/tests/integration/salesforce/salesforce_mces_golden.json b/metadata-ingestion/tests/integration/salesforce/salesforce_mces_golden.json index 90df0997495f03..98deb2a2a7ad2f 100644 --- a/metadata-ingestion/tests/integration/salesforce/salesforce_mces_golden.json +++ b/metadata-ingestion/tests/integration/salesforce/salesforce_mces_golden.json @@ -1354,7 +1354,7 @@ { "fieldPath": "Unique_Account", "nullable": true, - "description": "\\# Unique_Account", + "description": "\\# Unique_Account\n\nThis is the # description\n\n\\# Help Text", "type": { "type": { "com.linkedin.schema.StringType": {}