From 14648f43dc5e4ae0ec3f5f5624f749d25aea72de Mon Sep 17 00:00:00 2001 From: Kunal-kankriya <127090035+Kunal-kankriya@users.noreply.github.com> Date: Mon, 26 Feb 2024 19:36:45 +0530 Subject: [PATCH 1/4] fix(tests): Change secret name to avoid conflicts (#9917) --- smoke-test/tests/privileges/test_privileges.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/smoke-test/tests/privileges/test_privileges.py b/smoke-test/tests/privileges/test_privileges.py index a4c607dac89f2..cdb5ead71aebc 100644 --- a/smoke-test/tests/privileges/test_privileges.py +++ b/smoke-test/tests/privileges/test_privileges.py @@ -500,13 +500,12 @@ def test_privilege_to_create_and_manage_policies(): _ensure_cant_perform_action(user_session, create_policy, "createPolicy") -@pytest.mark.skip(reason="Functionality and test needs to be validated for correctness") @pytest.mark.dependency(depends=["test_healthchecks"]) def test_privilege_from_group_role_can_create_and_manage_secret(): (admin_user, admin_pass) = get_admin_credentials() admin_session = login_as(admin_user, admin_pass) user_session = login_as("user", "user") - secret_urn = "urn:li:dataHubSecret:TestSecretName" + secret_urn = "urn:li:dataHubSecret:TestName" # Verify new user can't create secrets create_secret = { @@ -514,7 +513,7 @@ def test_privilege_from_group_role_can_create_and_manage_secret(): createSecret(input: $input)\n}""", "variables": { "input": { - "name": "TestSecretName", + "name": "TestName", "value": "Test Secret Value", "description": "Test Secret Description", } From 21ecb7170f4687ac415ebc2b4a70a38a8f50c7c0 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Mon, 26 Feb 2024 15:57:01 +0000 Subject: [PATCH 2/4] feat(policies): Make policies searchable by privilege, type, status or editable fields (#9877) --- .../policy/ListPoliciesResolver.java | 21 +- .../src/main/resources/entity.graphql | 5 + .../config/BackfillPolicyFieldsConfig.java | 23 ++ .../upgrade/config/SystemUpdateConfig.java | 7 +- .../datahub/upgrade/system/SystemUpdate.java | 7 +- .../entity/steps/BackfillPolicyFields.java | 38 +++ .../steps/BackfillPolicyFieldsStep.java | 216 ++++++++++++++++++ .../com/linkedin/policy/DataHubPolicyInfo.pdl | 15 ++ .../authorization/DataHubAuthorizer.java | 2 +- .../datahub/authorization/PolicyFetcher.java | 18 +- .../src/main/resources/application.yml | 5 + 11 files changed, 345 insertions(+), 12 deletions(-) create mode 100644 datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillPolicyFieldsConfig.java create mode 100644 datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillPolicyFields.java create mode 100644 datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillPolicyFieldsStep.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java index 87832b8c3aa40..c1e4bb7f83316 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java @@ -5,17 +5,25 @@ import com.datahub.authorization.PolicyFetcher; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.AndFilterInput; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.ListPoliciesInput; import com.linkedin.datahub.graphql.generated.ListPoliciesResult; import com.linkedin.datahub.graphql.generated.Policy; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.resolvers.policy.mappers.PolicyInfoPolicyMapper; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.query.filter.Filter; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class ListPoliciesResolver implements DataFetcher> { private static final Integer DEFAULT_START = 0; @@ -40,9 +48,20 @@ public CompletableFuture get(final DataFetchingEnvironment e final Integer start = input.getStart() == null ? DEFAULT_START : input.getStart(); final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount(); final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery(); + final List filters = + input.getOrFilters() != null ? input.getOrFilters() : new ArrayList<>(); + final List facetFilters = + filters.stream() + .map(AndFilterInput::getAnd) + .flatMap(List::stream) + .collect(Collectors.toList()); + log.debug( + "User {} listing policies with filters {}", context.getActorUrn(), filters.toString()); + + final Filter filter = ResolverUtils.buildFilter(facetFilters, Collections.emptyList()); return _policyFetcher - .fetchPolicies(start, query, count, context.getAuthentication()) + .fetchPolicies(start, query, count, filter, context.getAuthentication()) .thenApply( policyFetchResult -> { final ListPoliciesResult result = new ListPoliciesResult(); diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index b5fd4a9a52a79..d8dedf20e3c0c 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -8682,6 +8682,11 @@ input ListPoliciesInput { Optional search query """ query: String + + """ + A list of disjunctive criterion for the filter. (or operation to combine filters) + """ + orFilters: [AndFilterInput!] } """ diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillPolicyFieldsConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillPolicyFieldsConfig.java new file mode 100644 index 0000000000000..6da85a5c16979 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillPolicyFieldsConfig.java @@ -0,0 +1,23 @@ +package com.linkedin.datahub.upgrade.config; + +import com.linkedin.datahub.upgrade.system.entity.steps.BackfillPolicyFields; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.search.SearchService; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class BackfillPolicyFieldsConfig { + + @Bean + public BackfillPolicyFields backfillPolicyFields( + EntityService entityService, + SearchService searchService, + @Value("${systemUpdate.policyFields.enabled}") final boolean enabled, + @Value("${systemUpdate.policyFields.reprocess.enabled}") final boolean reprocessEnabled, + @Value("${systemUpdate.policyFields.batchSize}") final Integer batchSize) { + return new BackfillPolicyFields( + entityService, searchService, enabled, reprocessEnabled, batchSize); + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java index cde3a29248fb5..17ad56ec80bac 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java @@ -4,6 +4,7 @@ import com.linkedin.datahub.upgrade.system.elasticsearch.BuildIndices; import com.linkedin.datahub.upgrade.system.elasticsearch.CleanIndices; import com.linkedin.datahub.upgrade.system.entity.steps.BackfillBrowsePathsV2; +import com.linkedin.datahub.upgrade.system.entity.steps.BackfillPolicyFields; import com.linkedin.datahub.upgrade.system.via.ReindexDataJobViaNodesCLL; import com.linkedin.gms.factory.common.TopicConventionFactory; import com.linkedin.gms.factory.config.ConfigurationProvider; @@ -40,7 +41,8 @@ public SystemUpdate systemUpdate( final GitVersion gitVersion, @Qualifier("revision") String revision, final BackfillBrowsePathsV2 backfillBrowsePathsV2, - final ReindexDataJobViaNodesCLL reindexDataJobViaNodesCLL) { + final ReindexDataJobViaNodesCLL reindexDataJobViaNodesCLL, + final BackfillPolicyFields backfillPolicyFields) { String version = String.format("%s-%s", gitVersion.getVersion(), revision); return new SystemUpdate( @@ -49,7 +51,8 @@ public SystemUpdate systemUpdate( kafkaEventProducer, version, backfillBrowsePathsV2, - reindexDataJobViaNodesCLL); + reindexDataJobViaNodesCLL, + backfillPolicyFields); } @Value("#{systemEnvironment['DATAHUB_REVISION'] ?: '0'}") diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/SystemUpdate.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/SystemUpdate.java index ed9c8ddda45c8..f02c820066c1c 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/SystemUpdate.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/SystemUpdate.java @@ -7,6 +7,7 @@ import com.linkedin.datahub.upgrade.system.elasticsearch.CleanIndices; import com.linkedin.datahub.upgrade.system.elasticsearch.steps.DataHubStartupStep; import com.linkedin.datahub.upgrade.system.entity.steps.BackfillBrowsePathsV2; +import com.linkedin.datahub.upgrade.system.entity.steps.BackfillPolicyFields; import com.linkedin.datahub.upgrade.system.via.ReindexDataJobViaNodesCLL; import com.linkedin.metadata.dao.producer.KafkaEventProducer; import java.util.List; @@ -26,11 +27,13 @@ public SystemUpdate( final KafkaEventProducer kafkaEventProducer, final String version, final BackfillBrowsePathsV2 backfillBrowsePathsV2, - final ReindexDataJobViaNodesCLL upgradeViaNodeCll) { + final ReindexDataJobViaNodesCLL upgradeViaNodeCll, + final BackfillPolicyFields backfillPolicyFields) { _preStartupUpgrades = List.of(buildIndicesJob); _steps = List.of(new DataHubStartupStep(kafkaEventProducer, version)); - _postStartupUpgrades = List.of(cleanIndicesJob, backfillBrowsePathsV2, upgradeViaNodeCll); + _postStartupUpgrades = + List.of(cleanIndicesJob, backfillBrowsePathsV2, upgradeViaNodeCll, backfillPolicyFields); } @Override diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillPolicyFields.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillPolicyFields.java new file mode 100644 index 0000000000000..3e1d385b87e45 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillPolicyFields.java @@ -0,0 +1,38 @@ +package com.linkedin.datahub.upgrade.system.entity.steps; + +import com.google.common.collect.ImmutableList; +import com.linkedin.datahub.upgrade.Upgrade; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.search.SearchService; +import java.util.List; + +public class BackfillPolicyFields implements Upgrade { + private final List _steps; + + public BackfillPolicyFields( + EntityService entityService, + SearchService searchService, + boolean enabled, + boolean reprocessEnabled, + Integer batchSize) { + if (enabled) { + _steps = + ImmutableList.of( + new BackfillPolicyFieldsStep( + entityService, searchService, reprocessEnabled, batchSize)); + } else { + _steps = ImmutableList.of(); + } + } + + @Override + public String id() { + return "BackfillPolicyFields"; + } + + @Override + public List steps() { + return _steps; + } +} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillPolicyFieldsStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillPolicyFieldsStep.java new file mode 100644 index 0000000000000..27d48aa5e0555 --- /dev/null +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillPolicyFieldsStep.java @@ -0,0 +1,216 @@ +package com.linkedin.datahub.upgrade.system.entity.steps; + +import static com.linkedin.metadata.Constants.*; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.DataMap; +import com.linkedin.datahub.upgrade.UpgradeContext; +import com.linkedin.datahub.upgrade.UpgradeStep; +import com.linkedin.datahub.upgrade.UpgradeStepResult; +import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; +import com.linkedin.entity.EntityResponse; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.boot.BootstrapStep; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.ScrollResult; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.policy.DataHubPolicyInfo; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.function.Function; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + +/** + * This bootstrap step is responsible for upgrading DataHub policy documents with new searchable + * fields in ES + */ +@Slf4j +public class BackfillPolicyFieldsStep implements UpgradeStep { + private static final String UPGRADE_ID = "BackfillPolicyFieldsStep"; + private static final Urn UPGRADE_ID_URN = BootstrapStep.getUpgradeUrn(UPGRADE_ID); + private final boolean reprocessEnabled; + private final Integer batchSize; + private final EntityService entityService; + private final SearchService _searchService; + + public BackfillPolicyFieldsStep( + EntityService entityService, + SearchService searchService, + boolean reprocessEnabled, + Integer batchSize) { + this.entityService = entityService; + this._searchService = searchService; + this.reprocessEnabled = reprocessEnabled; + this.batchSize = batchSize; + } + + @Override + public String id() { + return UPGRADE_ID; + } + + @Override + public Function executable() { + return (context) -> { + final AuditStamp auditStamp = + new AuditStamp() + .setActor(UrnUtils.getUrn(Constants.SYSTEM_ACTOR)) + .setTime(System.currentTimeMillis()); + + String scrollId = null; + int migratedCount = 0; + do { + log.info("Upgrading batch of policies {}-{}", migratedCount, migratedCount + batchSize); + scrollId = backfillPolicies(auditStamp, scrollId); + migratedCount += batchSize; + } while (scrollId != null); + + BootstrapStep.setUpgradeResult(UPGRADE_ID_URN, entityService); + + return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); + }; + } + + /** + * Returns whether the upgrade should proceed if the step fails after exceeding the maximum + * retries. + */ + @Override + public boolean isOptional() { + return true; + } + + /** + * Returns whether the upgrade should be skipped. Uses previous run history or the environment + * variables REPROCESS_DEFAULT_POLICY_FIELDS & BACKFILL_BROWSE_PATHS_V2 to determine whether to + * skip. + */ + @Override + public boolean skip(UpgradeContext context) { + + if (reprocessEnabled) { + return false; + } + + boolean previouslyRun = entityService.exists(UPGRADE_ID_URN, true); + if (previouslyRun) { + log.info("{} was already run. Skipping.", id()); + } + return previouslyRun; + } + + private String backfillPolicies(AuditStamp auditStamp, String scrollId) { + + final Filter filter = backfillPolicyFieldFilter(); + final ScrollResult scrollResult = + _searchService.scrollAcrossEntities( + ImmutableList.of(Constants.POLICY_ENTITY_NAME), + "*", + filter, + null, + scrollId, + null, + batchSize, + new SearchFlags() + .setFulltext(true) + .setSkipCache(true) + .setSkipHighlighting(true) + .setSkipAggregates(true)); + + if (scrollResult.getNumEntities() == 0 || scrollResult.getEntities().isEmpty()) { + return null; + } + + for (SearchEntity searchEntity : scrollResult.getEntities()) { + try { + ingestPolicyFields(searchEntity.getEntity(), auditStamp); + } catch (Exception e) { + // don't stop the whole step because of one bad urn or one bad ingestion + log.error( + String.format( + "Error ingesting default browsePathsV2 aspect for urn %s", + searchEntity.getEntity()), + e); + } + } + + return scrollResult.getScrollId(); + } + + private Filter backfillPolicyFieldFilter() { + // Condition: Does not have at least 1 of: `privileges`, `editable`, `state` or `type` + ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray(); + + conjunctiveCriterionArray.add(getCriterionForMissingField("privilege")); + conjunctiveCriterionArray.add(getCriterionForMissingField("editable")); + conjunctiveCriterionArray.add(getCriterionForMissingField("state")); + conjunctiveCriterionArray.add(getCriterionForMissingField("type")); + + Filter filter = new Filter(); + filter.setOr(conjunctiveCriterionArray); + return filter; + } + + private void ingestPolicyFields(Urn urn, AuditStamp auditStamp) { + EntityResponse entityResponse = null; + try { + entityResponse = + entityService.getEntityV2( + urn.getEntityType(), urn, Collections.singleton(DATAHUB_POLICY_INFO_ASPECT_NAME)); + } catch (URISyntaxException e) { + log.error( + String.format( + "Error getting DataHub Policy Info for entity with urn %s while restating policy information", + urn), + e); + } + + if (entityResponse != null + && entityResponse.getAspects().containsKey(DATAHUB_POLICY_INFO_ASPECT_NAME)) { + final DataMap dataMap = + entityResponse.getAspects().get(DATAHUB_POLICY_INFO_ASPECT_NAME).getValue().data(); + final DataHubPolicyInfo infoAspect = new DataHubPolicyInfo(dataMap); + log.debug("Restating policy information for urn {} with value {}", urn, infoAspect); + MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(urn); + proposal.setEntityType(urn.getEntityType()); + proposal.setAspectName(DATAHUB_POLICY_INFO_ASPECT_NAME); + proposal.setChangeType(ChangeType.RESTATE); + proposal.setSystemMetadata( + new SystemMetadata() + .setRunId(DEFAULT_RUN_ID) + .setLastObserved(System.currentTimeMillis())); + proposal.setAspect(GenericRecordUtils.serializeAspect(infoAspect)); + entityService.ingestProposal(proposal, auditStamp, true); + } + } + + @NotNull + private static ConjunctiveCriterion getCriterionForMissingField(String field) { + final Criterion missingPrivilegesField = new Criterion(); + missingPrivilegesField.setCondition(Condition.IS_NULL); + missingPrivilegesField.setField(field); + + final CriterionArray criterionArray = new CriterionArray(); + criterionArray.add(missingPrivilegesField); + final ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion(); + conjunctiveCriterion.setAnd(criterionArray); + return conjunctiveCriterion; + } +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/policy/DataHubPolicyInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/policy/DataHubPolicyInfo.pdl index 8e43e531ebfe1..503cadd5d7a47 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/policy/DataHubPolicyInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/policy/DataHubPolicyInfo.pdl @@ -27,11 +27,17 @@ record DataHubPolicyInfo { /** * The type of policy */ + @Searchable = { + "fieldType": "KEYWORD" + } type: string /** * The state of policy, ACTIVE or INACTIVE */ + @Searchable = { + "fieldType": "KEYWORD" + } state: string /** @@ -42,6 +48,12 @@ record DataHubPolicyInfo { /** * The privileges that the policy grants. */ + @Searchable = { + "/*": { + "fieldType": "KEYWORD", + "addToFilters": true + } + } privileges: array[string] /** @@ -52,6 +64,9 @@ record DataHubPolicyInfo { /** * Whether the policy should be editable via the UI */ + @Searchable = { + "fieldType": "BOOLEAN" + } editable: boolean = true /** diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java index 9ae95bd4e92b6..350d57aae3783 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DataHubAuthorizer.java @@ -278,7 +278,7 @@ public void run() { while (total == null || scrollId != null) { try { final PolicyFetcher.PolicyFetchResult policyFetchResult = - _policyFetcher.fetchPolicies(count, scrollId, _systemAuthentication); + _policyFetcher.fetchPolicies(count, scrollId, null, _systemAuthentication); addPoliciesToCache(newCache, policyFetchResult.getPolicies()); diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyFetcher.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyFetcher.java index 9c5950985eea4..0485e3000ad17 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyFetcher.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyFetcher.java @@ -9,6 +9,7 @@ import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; import com.linkedin.metadata.search.ScrollResult; @@ -45,7 +46,7 @@ public class PolicyFetcher { */ @Deprecated public CompletableFuture fetchPolicies( - int start, String query, int count, Authentication authentication) { + int start, String query, int count, Filter filter, Authentication authentication) { return CompletableFuture.supplyAsync( () -> { try { @@ -55,7 +56,8 @@ public CompletableFuture fetchPolicies( while (PolicyFetchResult.EMPTY.equals(result) && scrollId != null) { PolicyFetchResult tmpResult = - fetchPolicies(query, count, scrollId.isEmpty() ? null : scrollId, authentication); + fetchPolicies( + query, count, scrollId.isEmpty() ? null : scrollId, filter, authentication); fetchedResults += tmpResult.getPolicies().size(); scrollId = tmpResult.getScrollId(); if (fetchedResults > start) { @@ -71,13 +73,17 @@ public CompletableFuture fetchPolicies( } public PolicyFetchResult fetchPolicies( - int count, @Nullable String scrollId, Authentication authentication) + int count, @Nullable String scrollId, Filter filter, Authentication authentication) throws RemoteInvocationException, URISyntaxException { - return fetchPolicies("", count, scrollId, authentication); + return fetchPolicies("", count, scrollId, filter, authentication); } public PolicyFetchResult fetchPolicies( - String query, int count, @Nullable String scrollId, Authentication authentication) + String query, + int count, + @Nullable String scrollId, + Filter filter, + Authentication authentication) throws RemoteInvocationException, URISyntaxException { log.debug(String.format("Batch fetching policies. count: %s, scroll: %s", count, scrollId)); @@ -86,7 +92,7 @@ public PolicyFetchResult fetchPolicies( _entityClient.scrollAcrossEntities( List.of(POLICY_ENTITY_NAME), query, - null, + filter, scrollId, null, count, diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml index 29dd32a375c5e..494c18b75f023 100644 --- a/metadata-service/configuration/src/main/resources/application.yml +++ b/metadata-service/configuration/src/main/resources/application.yml @@ -322,6 +322,11 @@ systemUpdate: batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_BROWSE_PATHS_V2_BATCH_SIZE:5000} reprocess: enabled: ${REPROCESS_DEFAULT_BROWSE_PATHS_V2:false} + policyFields: + enabled: ${BOOTSTRAP_SYSTEM_UPDATE_POLICY_FIELDS_ENABLED:true} + batchSize: ${BOOTSTRAP_SYSTEM_UPDATE_POLICY_FIELDS_BATCH_SIZE:5000} + reprocess: + enabled: ${REPROCESS_DEFAULT_POLICY_FIELDS:false} structuredProperties: enabled: ${ENABLE_STRUCTURED_PROPERTIES_HOOK:true} # applies structured properties mappings From 93acb82b4c32dfd440257a8ebadec7ea9e0bb36b Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Tue, 27 Feb 2024 00:55:33 +0530 Subject: [PATCH 3/4] feat(ci): exempt more names from being marked as community (#9918) --- .github/workflows/pr-labeler.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 5a6978dfde19d..fc6bdb856816f 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -37,7 +37,13 @@ jobs: "maggiehays", "mrjefflewis", "pedro93", - "RyanHolstien" + "RyanHolstien", + "Kunal-kankriya", + "purnimagarg1", + "gaurav2733", + "dushayntAW", + "AvaniSiddhapuraAPT", + "akarsh991" ]'), github.actor ) From a1f2216da71c79ec8c111c96e0a47760fb943b9d Mon Sep 17 00:00:00 2001 From: Shubham Jagtap <132359390+shubhamjagtap639@users.noreply.github.com> Date: Tue, 27 Feb 2024 00:57:37 +0530 Subject: [PATCH 4/4] feat(ingest/qlik): Qlik cloud connector integration (#9682) Co-authored-by: Harshal Sheth --- .../app/ingest/source/builder/constants.ts | 4 + .../app/ingest/source/builder/sources.json | 7 + datahub-web-react/src/images/qliklogo.png | Bin 0 -> 7505 bytes .../docs/sources/qlik-sense/qlik-sense_pre.md | 23 + .../sources/qlik-sense/qlik-sense_recipe.yml | 25 + metadata-ingestion/setup.py | 3 + .../ingestion/source/common/subtypes.py | 3 + .../ingestion/source/qlik_sense/__init__.py | 0 .../ingestion/source/qlik_sense/config.py | 130 ++ .../source/qlik_sense/data_classes.py | 235 +++ .../ingestion/source/qlik_sense/qlik_api.py | 300 +++ .../ingestion/source/qlik_sense/qlik_sense.py | 602 ++++++ .../source/qlik_sense/websocket_connection.py | 55 + .../golden_test_platform_instance_ingest.json | 1680 +++++++++++++++++ .../golden_test_qlik_sense_ingest.json | 1517 +++++++++++++++ .../integration/qlik_sense/test_qlik_sense.py | 1088 +++++++++++ .../main/resources/boot/data_platforms.json | 10 + 17 files changed, 5682 insertions(+) create mode 100644 datahub-web-react/src/images/qliklogo.png create mode 100644 metadata-ingestion/docs/sources/qlik-sense/qlik-sense_pre.md create mode 100644 metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_sense/__init__.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/qlik_sense/websocket_connection.py create mode 100644 metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json create mode 100644 metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json create mode 100644 metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts index 3784201c71c0e..27674f6dc38e1 100644 --- a/datahub-web-react/src/app/ingest/source/builder/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts @@ -31,6 +31,7 @@ import mlflowLogo from '../../../../images/mlflowlogo.png'; import dynamodbLogo from '../../../../images/dynamodblogo.png'; import fivetranLogo from '../../../../images/fivetranlogo.png'; import csvLogo from '../../../../images/csv-logo.png'; +import qlikLogo from '../../../../images/qliklogo.png'; export const ATHENA = 'athena'; export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`; @@ -113,6 +114,8 @@ export const FIVETRAN = 'fivetran'; export const FIVETRAN_URN = `urn:li:dataPlatform:${FIVETRAN}`; export const CSV = 'csv-enricher'; export const CSV_URN = `urn:li:dataPlatform:${CSV}`; +export const QLIK_SENSE = 'qlik-sense'; +export const QLIK_SENSE_URN = `urn:li:dataPlatform:${QLIK_SENSE}`; export const PLATFORM_URN_TO_LOGO = { [ATHENA_URN]: athenaLogo, @@ -149,6 +152,7 @@ export const PLATFORM_URN_TO_LOGO = { [VERTICA_URN]: verticaLogo, [FIVETRAN_URN]: fivetranLogo, [CSV_URN]: csvLogo, + [QLIK_SENSE_URN]: qlikLogo, }; export const SOURCE_TO_PLATFORM_URN = { diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index e33de13c100b7..f4bfd45eb4f83 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -236,5 +236,12 @@ "displayName": "Other", "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/", "recipe": "source:\n type: \n config:\n # Source-type specifics config\n " + }, + { + "urn": "urn:li:dataPlatform:qlik-sense", + "name": "qlik-sense", + "displayName": "Qlik Sense", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/qlik-sense/", + "recipe": "source:\n type: qlik-sense\n config:\n # Coordinates\n tenant_hostname: https://xyz12xz.us.qlikcloud.com\n # Coordinates\n api_key: QLIK_API_KEY\n\n # Optional - filter for certain space names instead of ingesting everything.\n # space_pattern:\n\n # allow:\n # - space_name\n ingest_owner: true" } ] diff --git a/datahub-web-react/src/images/qliklogo.png b/datahub-web-react/src/images/qliklogo.png new file mode 100644 index 0000000000000000000000000000000000000000..fe69faed8b9a9215c1f3e3d9fef26beb501ef789 GIT binary patch literal 7505 zcmYj$c{o(>|Gs^T#2{o#jBJ@p?(;Ua!%z^Iz9lV5SWD z!R#USSHn5w&UN|LRLzJ9*ipgNW44>An%6T#mx}Zkqa-GbsLlBN6I;^@Kqi%-9==sC zoT`bH?+IRb9ao0ccTUSo7x90%$t%$7p9_DClpZ%aY@ON<<1gL$at)EtA_SXHVG6N3 zcV*{I)trw!t6Lo`DEpoEce7lH0si7rXuO$fZ!9J&x6$Cx2NkAz%54~wVMh_2fY$ms z@l<N=nuWym}&GSme8drL8A{yr7UgKT;=bEJ%yua_@Rz-cg#3P(IPGrw$FGlpob^hviyi?$lm;2opc4q(5THWOmaENi|;4G`mXH*bg+w4%%7>PRBX{1GUXl=2sZ@uI1#fS!TWYFfl*f za7t?Ksq7Hq!sFkrXcqI8q}j$){+mKLUDwUDS%sfgrL{##)%=dnJVQyrvJr-!g}zQR zKunQH66g-F>u2Yb?zywZR5Qc?)l^~9uNL^ur#&)%;7_i>%QuD5BU zRWKVpw+s|fp_<)~q94S82$k;+<~G>LLcLqaIRQ-QiffDg202J^&D<@Nh^S57f`(rA zE|jdo3bdTHXajroHX~xA2h(DUXQ1A@ioU_udQJjE>l2|B||S(#98M{=3@^$tLs%CmN@!s7(aV4_Cq z`dX&Ef&78}I1eox_DxO2`%=I94V{&oCl5MdvcRUVCmxXDT~ngOV+68D#KrjRczl@m z>I(ci3V0iYPTo={o<`6}#8B76zodZ4c2n?#izi{?09M+_%SA|6040s(w% zLs%7-ECGBNfu*7x(7vacMmS9pQ5ZSvOiJlS7g99p^O(@=Sd`!_5bBmKMWI5K&)Wl) z!xGb28EL?+QqVneUvYHD(~F}1WN+y z+!*y04r0igK)DU01<6q5UHk9Us|1#m9Pq*P@{;CDPwM&z+}}e6#TbM_!+4gI0)QBvQndw(vtPIEo;vZD9O*BDbOqMS zEPl0aN5^ICJaEfPMts*f1)pqqs=l|$(ZytvEq+~6lQX?+4Lquzorr3TU6sW`q;l<* z)OEj39mGl56P1!+!}9HFS}{16(K8ymu0QQ|5t(fszO1z|mHs1j+3ag2&mruTIe^8q zsZ$$LB2k~?xeNvlIzIVgddcVtqIoPcV!m<{49Vh1Ic5hV(W)`?n~LLh--zyJIPKew zJF-VOBNx(={@e~X8F_2MZeTt`HQ(Wkn3vYP0A40pk^Z-!i3H`RxbKY;qA z8dN0>nr-EbNTkL|Q$(OoCuRhzh5pmopPo!UjHnGDxFqMo^Z?I6wyy?w=R@>lMp|5E zpkh|Y{iLRkCDGgRK^CCaY!(DfS8f#7%46EcWOm!t*4Z+GRQmnYyt0A*)Bk{2bHq1w z!sIKl-jGl=RNT|p|86iy5}@DTio@Pv=s1>JuSszm5P)6v;`v~y9SXd4_!awOtnxum zWuq>sEiXl%;ZOZz^^7EN5h%_P{ml#dt?=x9yfB3<&kYnmCN>?Y;-m5F?%9a`IA{n5 zU%-m3>r`iu8sqhN9>L1Ep&u>=o`79O0_l!aDaXE%3-K&@QN+lI!aDl&iQEXCqt>nx z;4hh)$00|p38jJt6j4OqNaZhdJ)9S$4?Cg@+;zk;<-~@pkpAHD(r&+?7qw{H=OkfS zN}UYjeF&Ov==?^H>~UAP!T;b4WSXL3jRERR9bZosN-_kWZcC19OBtGg?KH$VgW7K; zIn2j@0UlBI`v^3B$!S9ilJ&BEXLI{|7sn|f6LNGz9m9!Q*?jvq{Bxi1P__W$c)+FA zAH%{w){N*%J>7*aO<67n+4S4?Mh;4UKKJ$$ruCexZd;0FG)lPaDQLE3Hm8>63s>-D zrse(~EDG1u%XiHDEmOgx#ZR!>VlMBUCbD-}S8^=1w*cL!t+nBsMFx>8HJ*}7}b^6f7=B$19vdmZQU zjkbqo-?z&iCChEqgp4EWO$EN14wY~bh#I;z*%p`&!*VO(A;fy}RK7rYjQcgRfgxs< z<7pe_2I9QZhm9ZIHJ(j7^*O>p8DvfDEQ9ozGF?%(zbA~I^l91F7xaG}TfIxOEy{V_EoFWKN>+)!c%rKe+_k zCVnjm1<-eel^kfPKym_<`vvNS-1DNv;)^g3KPdhr_>|P3{l_dZ1izXDO?{}wj%17< z`!v!0&@$U35UpgrCW!xPqs2~$=fF+a4^#N~UpNc`sfGQv3wj|+psj~j2`jW5Q@jOV9b zbHufaG4?d3pJ=X8JIDvuBtAsYku5<|1m1H8=2Cd!>?4y;j1#*M{=bGn`)v~&I|mG& zK(%gsn~e{7>yy-_=B?k!t-Pio$9|QceP2!`(nocm$ljyro;|~^-d~ar_xW#e#oZ9U zgMRz4U@d-Y%0`JzuGo9rR|8sJ2-Woqd-h#9vz-z1Cz5EJdB5>BxBG*EnqJAo3Z9z8 zMl!8GzkbZe=eKZIi55=$0uXK^J>oFsiY7hVBdT`Z(x|u_Ox(XO7yhOPusAQM*f%d? z`gjaZ4GZ+e%Fu)1w?1WPSYDGK6ODjzx9>xp`St*f+;<;_rZPwVNP3KGKo2Ws}-?jFo41 zQzXRtmBC|QecE9pU}%dVL|dO@w2oq1UC;;V5D-6@NFKTf%hF_kB{&FGUfc>`bSf)Q zU{LXlag$7f{W*qt1p44S#@q$HEDKpn6_Z&;x>F&S%YQlJ_@%7OTcsE>bI&zj?)95q zJ{d1390^wVhTe7l80o;UqavDB)n>$7fLWq{beFYXXAw_2T zicF{?lUa#1p$ag1_NV{<(E#3;O+}f2oe2|AF~hPJe)sov61}x?#4Culg0F}F+lln; zZ&(`%uU+$F?|e`H9JJXB+ChPO*U$dh$uQ?()M4SN_J((iz{mIdV(+_b`p?Xt;*|=n z{M6ALlcA-OX{>yVbxz5LXuZz$!$*_pT`^}gwQ)Ud&R@21yYn5X_IDSmf4SdV(EC(9 zKwmGSt8|J|p)p2a2z99_Ei-Pxf>VEp8541CG`FM1-5PI^@JA;)Z^0tLMA-HBu#`6) z@eSDHok@UfZGgiap!;=~nW|$L4=>t;8(3*s9vxZ`%l*Ch=G!d z1W9(*TtWYPs0Jot33-qYNI8y37e7YJzkRHoV}rR@DW^Zks`P`S=pjGT?!F#i<;*~i zakFWk{355}G=-&NEHd% zJpIxjBOzA^P8s%FEYh{x4df>r^5zbhe<;>hc6e24d?w|+*Y_v6*Y{}aOmP)4#7p-2 z=n77W`tDEQ^cx!mHW~f2|H`EPB4yr>(W1NGXyDE#xl#TvuLPbq6Y6{@bu;U4LsS?l zb;eGXycWbQ)5&vQKAPLCZ1>=!Xl?`6)L-`=r%MApIgeay4jsLhV^VRywyVtI1<$uZ z0ekh>(E=YuMv15j=f;1G^RzkbSmT;&l%x{`Z#f42yT6~&b~yP24Q7B5G)QkRiKO69 z(5@{V2{WhNh@wuy_o;EMF#!YJeD!@Qp0#=dQwqTw3{_iZ=KhFnR2L4!57PeOrL@>B zdr9t$4Hz>zciy%fBS?En+7fK*DuDrp+=4hOjVdXepOIs3z8V-@+EMunB8GTvU|FgW z=z2_LAldgx_R{xQRCI~+i; zA;CJ{3V&rijxpW|C6<^sZHKEwBtUzX<*xAbW4k(^gLurq zv3Rh`FM|hrh_f=>kMCPvA~6Ce?UK05efi)Bp-0iad{6w*m*aCeu#D%H{bR~pIsH;p zgKJJ8H%il7Ghcf^*bTbQN?yq+o-!XUQrrd=Z8pC;7vRj&iQZyM$l9BM8-80J&D>^TGfDh%jk)j)!*JRNt z>HZ%6tCLYrD-CGjd=cy$@SXZPBn9r1Ut0?Kwh=eR^YY?lMKEK=$&0Cn4Rj1zZe5Kz z2sVAt~q|@hfp4yY5rAFo-*s*f74L|6Fi8SI5Fi#`wzpeQZdClnPr5w%WG%x{J7rn(I@- zUACOGU;b5nJE2J-a8JmN&?nog1V->Zih8m5<|>skv1>t#;q4jl8vcjEKD<$?zN9(x zY3cq}8yC%H@KUa&M0#$_bMdPqduJcTNblTMnKh^sWP4s|u-=yW${pak`k}#6TZ#)5 z^m*ct$ko;_je0i1(tENEN&uCUs=T~ay~{hyJ6-*FT|9SPf?Q!8w$_d`)qg}86 zxOK!${6|)I6n|~P4e{<&<2&70zj#lW4LhlTkvG!T=B6%cli6P<(DzuZwCMP)mUABb z%y-&|BXR9|3a$=v zv;iuPzr%Cl+v*A4qmZKoU~&BE6MZJo&-h~7$sBrj+buWuB=6^!rd;GjL) zV2F_xd@KZLy?^nJ89vDpc8*wiW}L!{Y~3Ga`@PV1}F<37KC zXXy2NDe?C&S06xVn^NyAwgQ=_+l-Hwgn>G0(06vF@&EB+(o`@?VS!D3ct+PKNKRG%A$TTBwPWYum?OmU~f7t?q2eF#Yn9;Xg zwW^=15I-STR}T4)wR9JhS95`1z_?HO^{rL%f>#p+MPg0igIWxo`hu3N3KQ=rqOtyI z5m4!?_r)t)5v~s`x>rz7;E-2&%rcDQ%f-gQRevifobTtH=Q?Q~m0UO`lXH_S+&?LG zb_;URF`#GMYy7jc*-ywREo&Lp{IRr!q-%iKq}kxPK*y9(*<;Gor!ZV3`m1*MEKE;{ z`gF`P#3*?cERZRdI#VKQLu|q)qSg4Ill_ zyw^^09{T2-NV@^FdL)0~*qboo^|{F&>uL~*?U3L=Wye~cf17<8x$*d>stncj)V9l$ z>;uelSed$42gHrSl&taxw1z&e+rSZGBq}r=4NwXYOi=$mm-Ua6=b7xbq+^03Bb0gj z6Us-Mv#3}8Es(W?Gr(G%dFyJ43FWN*+ez;GFEUg-q+dwXPnK32}p{%mKr0S)jBWWo%~e(XTNR( z+oxJ!XZEov<|8rJ(yhvys!~*kHD5}~%d+<6eGd(FFZ|HLqD$$P=M(7z`w_>HuOM~R z|97?hf2M8}@qgY6zSlL?%`!w?hew=(qW_otx{W@=mYKC*iwsEJv(083?SJa!umJl0 zE=ECbA%+5!F3U_zaJ0VW+vI4G5R-Q&bIi#>-G*h=)+r<PIG+n@Fq0%%>$~?MnBnJSiJH z?flPR>hLmcBioy1N?gG{2ZIlbx`w1|Y<4J)to!d*3}>J-C41NtpPZmcI}{gb8?%oU ze{7DSaz%32U_^BILQSXScCFwHTgt~{;`t%B7u;KzwE~Akf@Sw>GFAozh=JY~_3H3y95NM1=1-T}ruAAE)FF z)L@dYeR!M+iJ;5~2IO%v7*!2`cyoWj4_cy6?F{Tot(FtU|A$G4iSC=Ig31Lz5 za`@JkB$EqTLo4p}KVlnp+-i#(?H7N10?@%vWT-Wnz!Bf?^}oD<&bo*~iTYcILd=@1 zt&Hz!Qno#n5n*SWL*AI3NFv2qTwe9-8yLlz=Z}$v_a7DcwM6sojMg~TsHx(7g*X)p zViuxsI+(~jc{avdX_wKTepdI}dF?(=ddPGA)1OQVA*hjuVv|gRuXpSVAbIn6+kvt` zTMsCr!Dz@RN|?Y7y*LIbSbHE)*U3?5B#{y(FeVKONbd-T6X$KbX3&315livvV^3Y+ z@5Bk3y>drp&OYujY`-=nnQ9;tV_H2d%{D=gZgz2gvWn@(QH^w;1BjVF9Sd_Z3xv?# z8jcrQ;I!-IOJ?$qj2X$KgqNzowV7t6Lz;zzJUNqUAzYAsmmGJMeGzD{@&I*B;$r^Q zxO_qmXs>TqW}I=f=4-7*5o%bT`hjHnX(gGIv}lZ-C4J48=M@XfhKNhVqy=E6zvln; z0j^(;mvaeTiL}fmc!^4R9AGvHD3mF_Wamrqrf4S+VV8=lFQq0DoYL9zKPR&3*TnKo z#E)|5Y2s;NlaMs)^jd?1DZK@RJS~P>r;eToi`QZ2)9*6flgAlrFD(nN{+I$GYPQQS zJ%PjoYAq=g{`{ty1ZMqp5p@Sn<_=&k_X1`1qxP+7B-?o7s(bLG=Zc~KQI8h--}@RM-clwjISCISUM_VuiA-BuO#}MjD zqvBUeabs_6pQbr+{ass5Oucr<0Vfl0^ z<6DxvEL)6-Vvk4f87XS>w6&=J*?7&!@z!q++K8miy{)Sf@tRj}DD$$4A?o7CdI&2b zgxu*WSm;wofVlUgK*89)vi3Rg?T-P2k{;FH@sXN-j$3;b-Yx}r|*BL{_%I?{Lkg)j-~yd Qrd(Y1XPj(aoJQaKf2g%Mr2qf` literal 0 HcmV?d00001 diff --git a/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_pre.md b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_pre.md new file mode 100644 index 0000000000000..c344a32d07676 --- /dev/null +++ b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_pre.md @@ -0,0 +1,23 @@ +## Integration Details + +This source extracts the following: + +- Accessible spaces and apps within that spaces as Container. +- Qlik Datasets as Datahub Datasets with schema metadata. +- Sheets as Datahub dashboard and charts present inside sheets. + +## Configuration Notes + +1. Refer [doc](https://qlik.dev/authenticate/api-key/generate-your-first-api-key/) to generate an API key from the hub. +2. Get tenant hostname from About tab after login to qlik sense account. + +## Concept mapping + +| Qlik Sense | Datahub | Notes | +|------------------------|---------------------------------------------------------------|----------------------------------| +| `Space` | [Container](../../metamodel/entities/container.md) | SubType `"Qlik Space"` | +| `App` | [Container](../../metamodel/entities/container.md) | SubType `"Qlik App"` | +| `Sheet` | [Dashboard](../../metamodel/entities/dashboard.md) | | +| `Chart` | [Chart](../../metamodel/entities/chart.md) | | +| `Dataset` | [Dataset](../../metamodel/entities/dataset.md) | SubType `"Qlik Dataset"` | +| `User` | [User (a.k.a CorpUser)](../../metamodel/entities/corpuser.md) | Optionally Extracted | \ No newline at end of file diff --git a/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml new file mode 100644 index 0000000000000..315e3d1de3f7f --- /dev/null +++ b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml @@ -0,0 +1,25 @@ +source: + type: qlik-sense + config: + # Coordinates + tenant_hostname: "xyz12xz.us.qlikcloud.com" + # Credentials + api_key: "QLIK_API_KEY" + + # Optional - filter for certain space names instead of ingesting everything. + # Mention 'personal_space' if entities of personal space need to ingest + # space_pattern: + # allow: + # - space_name + + ingest_owner: true + + # Optional -- This mapping is optional and only required to configure platform-instance for Qlik app dataset upstream source tables + # A mapping of the Qlik app dataset upstream tables from data connection to platform instance. Use 'data_connection_name' as key. + # data_connection_to_platform_instance: + # data_connection_name: + # platform_instance: cloud_instance + # env: DEV + +sink: + # sink configs diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 6f0e4d8fb6f37..3c6aafec26185 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -393,6 +393,7 @@ # databricks is alias for unity-catalog and needs to be kept in sync "databricks": databricks | sql_common | sqllineage_lib, "fivetran": snowflake_common | bigquery_common, + "qlik-sense": sqlglot_lib | {"requests", "websocket-client"}, } # This is mainly used to exclude plugins from the Docker image. @@ -521,6 +522,7 @@ "mode", "fivetran", "kafka-connect", + "qlik-sense", ] if plugin for dependency in plugins[plugin] @@ -625,6 +627,7 @@ "gcs = datahub.ingestion.source.gcs.gcs_source:GCSSource", "sql-queries = datahub.ingestion.source.sql_queries:SqlQueriesSource", "fivetran = datahub.ingestion.source.fivetran.fivetran:FivetranSource", + "qlik-sense = datahub.ingestion.source.qlik_sense.qlik_sense:QlikSenseSource", ], "datahub.ingestion.transformer.plugins": [ "simple_remove_dataset_ownership = datahub.ingestion.transformer.remove_dataset_ownership:SimpleRemoveDatasetOwnership", diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py index 3296a8fb29354..1def0dd02097b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py @@ -15,6 +15,7 @@ class DatasetSubTypes(str, Enum): SALESFORCE_CUSTOM_OBJECT = "Custom Object" SALESFORCE_STANDARD_OBJECT = "Object" POWERBI_DATASET_TABLE = "PowerBI Dataset Table" + QLIK_DATASET = "Qlik Dataset" BIGQUERY_TABLE_SNAPSHOT = "Bigquery Table Snapshot" # TODO: Create separate entity... @@ -40,6 +41,8 @@ class BIContainerSubTypes(str, Enum): TABLEAU_WORKBOOK = "Workbook" POWERBI_WORKSPACE = "Workspace" POWERBI_DATASET = "PowerBI Dataset" + QLIK_SPACE = "Qlik Space" + QLIK_APP = "Qlik App" class BIAssetSubTypes(str, Enum): diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py new file mode 100644 index 0000000000000..eb0b9d02d865e --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py @@ -0,0 +1,130 @@ +import logging +from dataclasses import dataclass +from typing import Dict, Optional + +import pydantic + +from datahub.configuration.common import AllowDenyPattern +from datahub.configuration.source_common import ( + EnvConfigMixin, + PlatformInstanceConfigMixin, +) +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalSourceReport, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionConfigBase, +) + +logger = logging.getLogger(__name__) + +QLIK_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + + +class Constant: + """ + keys used in qlik plugin + """ + + # Rest API response key constants + DATA = "data" + ID = "id" + NAME = "name" + TYPE = "type" + ITEMID = "itemId" + NEXT = "next" + LINKS = "links" + HREF = "href" + DATASETTYPE = "datasetType" + CREATEDAT = "createdAt" + UPDATEDAT = "updatedAt" + SECUREQRI = "secureQri" + QRI = "qri" + SPACEID = "spaceId" + SPACE = "space" + CREATEDTIME = "createdTime" + LASTMODIFIEDTIME = "lastModifiedTime" + OPERATIONAL = "operational" + SIZE = "size" + ROWCOUNT = "rowCount" + DATATYPE = "dataType" + PRIMARYKEY = "primaryKey" + NULLABLE = "nullable" + SCHEMA = "schema" + DATAFIELDS = "dataFields" + RESOURCETYPE = "resourceType" + USAGE = "usage" + CREATEDDATE = "createdDate" + MODIFIEDDATE = "modifiedDate" + RESOURCEID = "resourceId" + DATASETSCHEMA = "datasetSchema" + GRAPH = "graph" + NODES = "nodes" + RESOURCES = "resources" + LINEAGE = "lineage" + TABLELABEL = "tableLabel" + TABLEQRI = "tableQRI" + OWNERID = "ownerId" + # Websocket response key constants + QID = "qId" + RESULT = "result" + QRETURN = "qReturn" + QTYPE = "qType" + QHANDLE = "qHandle" + QLAYOUT = "qLayout" + QMETA = "qMeta" + QCHILDLIST = "qChildList" + QITEMS = "qItems" + QINFO = "qInfo" + QLIST = "qList" + CONNECTORPROPERTIES = "connectorProperties" + TABLEQUALIFIERS = "tableQualifiers" + CONNECTIONINFO = "connectionInfo" + SOURCECONNECTORID = "sourceConnectorID" + DATABASENAME = "databaseName" + SCHEMANAME = "schemaName" + TABLES = "tables" + DATACONNECTORID = "dataconnectorid" + DATACONNECTORNAME = "dataconnectorName" + DATACONNECTORPLATFORM = "dataconnectorPlatform" + # Item type + APP = "app" + DATASET = "dataset" + # Personal entity constants + PERSONAL_SPACE_ID = "personal-space-id" + PERSONAL_SPACE_NAME = "personal_space" + + +@dataclass +class QlikSourceReport(StaleEntityRemovalSourceReport): + number_of_spaces: int = 0 + + def report_number_of_spaces(self, number_of_spaces: int) -> None: + self.number_of_spaces = number_of_spaces + + +class PlatformDetail(PlatformInstanceConfigMixin, EnvConfigMixin): + pass + + +class QlikSourceConfig( + StatefulIngestionConfigBase, PlatformInstanceConfigMixin, EnvConfigMixin +): + tenant_hostname: str = pydantic.Field(description="Qlik Tenant hostname") + api_key: str = pydantic.Field(description="Qlik API Key") + # Qlik space identifier + space_pattern: AllowDenyPattern = pydantic.Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns to filter Qlik spaces in ingestion." + "Mention 'personal_space' if entities of personal space need to ingest", + ) + ingest_owner: Optional[bool] = pydantic.Field( + default=True, + description="Ingest Owner from source. This will override Owner info entered from UI", + ) + # Qlik app dataset upstream tables from data connection to platform instance mapping + data_connection_to_platform_instance: Dict[str, PlatformDetail] = pydantic.Field( + default={}, + description="A mapping of the Qlik app dataset upstream tables from data connection to platform instance." + "Use 'data_connection_name' as key.", + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py new file mode 100644 index 0000000000000..c30b456253e06 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py @@ -0,0 +1,235 @@ +from datetime import datetime +from enum import Enum +from typing import Dict, List, Optional, Type, Union + +from pydantic import BaseModel, Field, root_validator + +from datahub.emitter.mcp_builder import ContainerKey +from datahub.ingestion.source.qlik_sense.config import QLIK_DATETIME_FORMAT, Constant +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + BooleanType, + BytesType, + DateType, + NumberType, + StringType, + TimeType, +) + +FIELD_TYPE_MAPPING: Dict[ + str, + Type[ + Union[ + BooleanType, + BytesType, + DateType, + NumberType, + StringType, + TimeType, + ] + ], +] = { + "DATE": DateType, + "TIME": TimeType, + "DATETIME": DateType, + "TIMESTAMP": DateType, + "STRING": StringType, + "DOUBLE": NumberType, + "DECIMAL": NumberType, + "INTEGER": NumberType, + "BOOLEAN": BooleanType, + "BINARY": BytesType, +} + +KNOWN_DATA_PLATFORM_MAPPING = { + "gbq": "bigquery", + "snowflake": "snowflake", +} + + +class SpaceKey(ContainerKey): + space: str + + +class AppKey(ContainerKey): + app: str + + +class SpaceType(Enum): + PERSONAL = "personal" + SHARED = "shared" + MANAGED = "managed" + DATA = "data" + + +# Qlik table box type +class BoxType(Enum): + LOADFILE = "load-file" # Table extracted from dataset + BLACKBOX = "blackbox" # Table extracted from data connection + + +PERSONAL_SPACE_DICT = { + "id": Constant.PERSONAL_SPACE_ID, + "name": Constant.PERSONAL_SPACE_NAME, + "description": "", + "type": SpaceType.PERSONAL, + "createdAt": datetime.now().strftime(QLIK_DATETIME_FORMAT), + "updatedAt": datetime.now().strftime(QLIK_DATETIME_FORMAT), +} + + +class Space(BaseModel): + id: str + name: str + description: str + type: SpaceType + createdAt: datetime + updatedAt: datetime + ownerId: Optional[str] = None + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.CREATEDAT] = datetime.strptime( + values[Constant.CREATEDAT], QLIK_DATETIME_FORMAT + ) + values[Constant.UPDATEDAT] = datetime.strptime( + values[Constant.UPDATEDAT], QLIK_DATETIME_FORMAT + ) + return values + + +class Item(BaseModel): + id: str + description: str = "" + ownerId: str + spaceId: str + createdAt: datetime + updatedAt: datetime + + +class SchemaField(BaseModel): + name: str + dataType: Optional[str] = None + primaryKey: Optional[bool] = None + nullable: Optional[bool] = None + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.DATATYPE] = values.get(Constant.DATATYPE, {}).get(Constant.TYPE) + return values + + +class QlikDataset(Item): + name: str + secureQri: str + type: str + size: int + rowCount: int + itemId: str + datasetSchema: List[SchemaField] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + # Update str time to datetime + values[Constant.CREATEDAT] = datetime.strptime( + values[Constant.CREATEDTIME], QLIK_DATETIME_FORMAT + ) + values[Constant.UPDATEDAT] = datetime.strptime( + values[Constant.LASTMODIFIEDTIME], QLIK_DATETIME_FORMAT + ) + if not values.get(Constant.SPACEID): + # spaceId none indicates dataset present in personal space + values[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID + values[Constant.QRI] = values[Constant.SECUREQRI] + values[Constant.SIZE] = values[Constant.OPERATIONAL].get(Constant.SIZE, 0) + values[Constant.ROWCOUNT] = values[Constant.OPERATIONAL][Constant.ROWCOUNT] + + values[Constant.DATASETSCHEMA] = values[Constant.SCHEMA][Constant.DATAFIELDS] + return values + + +class AxisProperty(BaseModel): + Title: str = Field(alias="qFallbackTitle") + Min: str = Field(alias="qMin") + Max: str = Field(alias="qMax") + + +class Chart(BaseModel): + qId: str + visualization: str + title: str + subtitle: str + qDimension: List[AxisProperty] + qMeasure: List[AxisProperty] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.QID] = values[Constant.QINFO][Constant.QID] + values["qDimension"] = values["qHyperCube"]["qDimensionInfo"] + values["qMeasure"] = values["qHyperCube"]["qMeasureInfo"] + return values + + +class Sheet(BaseModel): + id: str + title: str + description: str + ownerId: str + createdAt: datetime + updatedAt: datetime + charts: List[Chart] = [] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.CREATEDAT] = datetime.strptime( + values[Constant.CREATEDDATE], QLIK_DATETIME_FORMAT + ) + values[Constant.UPDATEDAT] = datetime.strptime( + values[Constant.MODIFIEDDATE], QLIK_DATETIME_FORMAT + ) + return values + + +class QlikTable(BaseModel): + tableName: str + type: BoxType = Field(alias="boxType") + tableAlias: str + dataconnectorid: str + dataconnectorName: str + dataconnectorPlatform: str + spaceId: str + datasetSchema: List[SchemaField] = Field(alias="fields") + tableQri: Optional[str] = None + selectStatement: Optional[str] = None + databaseName: Optional[str] = None + schemaName: Optional[str] = None + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.DATACONNECTORID] = values[Constant.CONNECTIONINFO][Constant.ID] + values[Constant.DATACONNECTORPLATFORM] = values[Constant.CONNECTIONINFO][ + Constant.SOURCECONNECTORID + ] + values[Constant.SPACEID] = values[Constant.CONNECTIONINFO][Constant.SPACE] + return values + + +class App(Item): + qTitle: str + qri: str + qUsage: str + sheets: List[Sheet] = [] + tables: List[QlikTable] = [] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.CREATEDAT] = datetime.strptime( + values[Constant.CREATEDDATE], QLIK_DATETIME_FORMAT + ) + values[Constant.UPDATEDAT] = datetime.strptime( + values[Constant.MODIFIEDDATE], QLIK_DATETIME_FORMAT + ) + if not values.get(Constant.SPACEID): + # spaceId none indicates app present in personal space + values[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID + values[Constant.QRI] = f"qri:app:sense://{values['id']}" + return values diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py new file mode 100644 index 0000000000000..abcd0c0ce8353 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py @@ -0,0 +1,300 @@ +import logging +import sys +from typing import Any, Dict, List, Optional +from urllib.parse import quote + +import requests + +from datahub.ingestion.source.qlik_sense.config import Constant, QlikSourceConfig +from datahub.ingestion.source.qlik_sense.data_classes import ( + PERSONAL_SPACE_DICT, + App, + Chart, + Item, + QlikDataset, + QlikTable, + Sheet, + Space, +) +from datahub.ingestion.source.qlik_sense.websocket_connection import WebsocketConnection + +# Logger instance +logger = logging.getLogger(__name__) + + +class QlikAPI: + def __init__(self, config: QlikSourceConfig) -> None: + self.spaces: Dict = {} + self.users: Dict = {} + self.config = config + self.session = requests.Session() + self.session.headers.update( + { + "Authorization": f"Bearer {self.config.api_key}", + "Content-Type": "application/json", + } + ) + self.rest_api_url = f"https://{self.config.tenant_hostname}/api/v1" + # Test connection by fetching list of api keys + logger.info("Trying to connect to {}".format(self.rest_api_url)) + self.session.get(f"{self.rest_api_url}/api-keys").raise_for_status() + + def _log_http_error(self, message: str) -> Any: + logger.warning(message) + _, e, _ = sys.exc_info() + if isinstance(e, requests.exceptions.HTTPError): + logger.warning(f"HTTP status-code = {e.response.status_code}") + logger.debug(msg=message, exc_info=e) + return e + + def get_spaces(self) -> List[Space]: + spaces: List[Space] = [] + try: + response = self.session.get(f"{self.rest_api_url}/spaces") + response.raise_for_status() + for space_dict in response.json()[Constant.DATA]: + space = Space.parse_obj(space_dict) + spaces.append(space) + self.spaces[space.id] = space.name + # Add personal space entity + spaces.append(Space.parse_obj(PERSONAL_SPACE_DICT)) + self.spaces[PERSONAL_SPACE_DICT[Constant.ID]] = PERSONAL_SPACE_DICT[ + Constant.NAME + ] + except Exception as e: + self._log_http_error(message=f"Unable to fetch spaces. Exception: {e}") + return spaces + + def _get_dataset(self, dataset_id: str, item_id: str) -> Optional[QlikDataset]: + try: + response = self.session.get(f"{self.rest_api_url}/data-sets/{dataset_id}") + response.raise_for_status() + response_dict = response.json() + response_dict[Constant.ITEMID] = item_id + return QlikDataset.parse_obj(response_dict) + except Exception as e: + self._log_http_error( + message=f"Unable to fetch dataset with id {dataset_id}. Exception: {e}" + ) + return None + + def get_user_name(self, user_id: str) -> Optional[str]: + try: + if user_id in self.users: + # To avoid fetching same user details again + return self.users[user_id] + else: + response = self.session.get(f"{self.rest_api_url}/users/{user_id}") + response.raise_for_status() + user_name = response.json()[Constant.NAME] + self.users[user_id] = user_name + return user_name + except Exception as e: + self._log_http_error( + message=f"Unable to fetch user with id {user_id}. Exception: {e}" + ) + return None + + def _get_chart( + self, + websocket_connection: WebsocketConnection, + chart_id: str, + sheet_id: str, + ) -> Optional[Chart]: + try: + websocket_connection.websocket_send_request( + method="GetChild", params={"qId": chart_id} + ) + response = websocket_connection.websocket_send_request(method="GetLayout") + return Chart.parse_obj(response[Constant.QLAYOUT]) + except Exception as e: + self._log_http_error( + message=f"Unable to fetch chart {chart_id} of sheet {sheet_id}. Exception: {e}" + ) + return None + + def _get_sheet( + self, + websocket_connection: WebsocketConnection, + sheet_id: str, + ) -> Optional[Sheet]: + try: + websocket_connection.websocket_send_request( + method="GetObject", params={"qId": sheet_id} + ) + response = websocket_connection.websocket_send_request(method="GetLayout") + sheet_dict = response[Constant.QLAYOUT] + if Constant.OWNERID not in sheet_dict[Constant.QMETA]: + # That means sheet is private sheet + return None + sheet = Sheet.parse_obj(sheet_dict[Constant.QMETA]) + for i, chart_dict in enumerate( + sheet_dict[Constant.QCHILDLIST][Constant.QITEMS] + ): + chart = self._get_chart( + websocket_connection, + chart_dict[Constant.QINFO][Constant.QID], + sheet_id, + ) + if chart: + if not chart.title: + chart.title = f"Object {i+1} of Sheet '{sheet.title}'" + sheet.charts.append(chart) + websocket_connection.handle.pop() + return sheet + except Exception as e: + self._log_http_error( + message=f"Unable to fetch sheet with id {sheet_id}. Exception: {e}" + ) + return None + + def _add_qri_of_tables(self, tables: List[QlikTable], app_id: str) -> None: + table_qri_dict: Dict[str, str] = {} + app_qri = quote(f"qri:app:sense://{app_id}", safe="") + try: + response = self.session.get( + f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/actions/expand?node={app_qri}&level=TABLE" + ) + response.raise_for_status() + for table_node_qri in response.json()[Constant.GRAPH][Constant.NODES]: + table_node_qri = quote(table_node_qri, safe="") + response = self.session.get( + f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/actions/expand?node={table_node_qri}&level=FIELD" + ) + response.raise_for_status() + field_nodes_qris = list( + response.json()[Constant.GRAPH][Constant.NODES].keys() + ) + for field_node_qri in field_nodes_qris: + response = self.session.post( + f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/overview", + json=[field_node_qri], + ) + response.raise_for_status() + # Some fields might not have lineage overview, in that case status code is 207 + if response.status_code == 200: + for each_lineage in response.json()[Constant.RESOURCES][0][ + Constant.LINEAGE + ]: + table_name = ( + each_lineage[Constant.TABLELABEL] + .replace('"', "") + .split(".")[-1] + ) + table_qri_dict[table_name] = each_lineage[Constant.TABLEQRI] + break + for table in tables: + if table.tableName in table_qri_dict: + table.tableQri = table_qri_dict[table.tableName] + except Exception as e: + self._log_http_error( + message=f"Unable to add QRI for tables of app {app_id}. Exception: {e}" + ) + + def _get_app_used_tables( + self, websocket_connection: WebsocketConnection, app_id: str + ) -> List[QlikTable]: + tables: List[QlikTable] = [] + try: + response = websocket_connection.websocket_send_request( + method="GetObject", + params=["LoadModel"], + ) + if not response[Constant.QRETURN][Constant.QTYPE]: + return [] + response = websocket_connection.websocket_send_request(method="GetLayout") + for table_dict in response[Constant.QLAYOUT][Constant.TABLES]: + tables.append(QlikTable.parse_obj(table_dict)) + websocket_connection.handle.pop() + self._add_qri_of_tables(tables, app_id) + except Exception as e: + self._log_http_error( + message=f"Unable to fetch tables used by app {app_id}. Exception: {e}" + ) + return tables + + def _get_app_sheets( + self, websocket_connection: WebsocketConnection, app_id: str + ) -> List[Sheet]: + sheets: List[Sheet] = [] + try: + response = websocket_connection.websocket_send_request( + method="GetObjects", + params={ + "qOptions": { + "qTypes": ["sheet"], + } + }, + ) + for sheet_dict in response[Constant.QLIST]: + sheet = self._get_sheet( + websocket_connection=websocket_connection, + sheet_id=sheet_dict[Constant.QINFO][Constant.QID], + ) + if sheet: + sheets.append(sheet) + websocket_connection.handle.pop() + except Exception as e: + self._log_http_error( + message=f"Unable to fetch sheets for app {app_id}. Exception: {e}" + ) + return sheets + + def _get_app(self, app_id: str) -> Optional[App]: + try: + websocket_connection = WebsocketConnection( + self.config.tenant_hostname, self.config.api_key, app_id + ) + websocket_connection.websocket_send_request( + method="OpenDoc", + params={"qDocName": app_id}, + ) + response = websocket_connection.websocket_send_request( + method="GetAppLayout" + ) + app = App.parse_obj(response[Constant.QLAYOUT]) + app.sheets = self._get_app_sheets(websocket_connection, app_id) + app.tables = self._get_app_used_tables(websocket_connection, app_id) + websocket_connection.close_websocket() + return app + except Exception as e: + self._log_http_error( + message=f"Unable to fetch app with id {app_id}. Exception: {e}" + ) + return None + + def get_items(self) -> List[Item]: + items: List[Item] = [] + try: + url = f"{self.rest_api_url}/items" + while True: + response = self.session.get(url) + response.raise_for_status() + response_dict = response.json() + for item in response_dict[Constant.DATA]: + # spaceId none indicates item present in personal space + if not item.get(Constant.SPACEID): + item[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID + if self.config.space_pattern.allowed( + self.spaces[item[Constant.SPACEID]] + ): + resource_type = item[Constant.RESOURCETYPE] + if resource_type == Constant.APP: + app = self._get_app(app_id=item[Constant.RESOURCEID]) + if app: + items.append(app) + elif resource_type == Constant.DATASET: + dataset = self._get_dataset( + dataset_id=item[Constant.RESOURCEID], + item_id=item[Constant.ID], + ) + if dataset: + items.append(dataset) + if Constant.NEXT in response_dict[Constant.LINKS]: + url = response_dict[Constant.LINKS][Constant.NEXT][Constant.HREF] + else: + break + + except Exception as e: + self._log_http_error(message=f"Unable to fetch items. Exception: {e}") + return items diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py new file mode 100644 index 0000000000000..a5b9adae0376c --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py @@ -0,0 +1,602 @@ +import logging +from datetime import datetime +from typing import Dict, Iterable, List, Optional + +import datahub.emitter.mce_builder as builder +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp_builder import add_entity_to_container, gen_containers +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.decorators import ( + SourceCapability, + SupportStatus, + capability, + config_class, + platform_name, + support_status, +) +from datahub.ingestion.api.source import ( + CapabilityReport, + MetadataWorkUnitProcessor, + SourceReport, + TestableSource, + TestConnectionReport, +) +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.common.subtypes import ( + BIContainerSubTypes, + DatasetSubTypes, +) +from datahub.ingestion.source.qlik_sense.config import ( + Constant, + PlatformDetail, + QlikSourceConfig, + QlikSourceReport, +) +from datahub.ingestion.source.qlik_sense.data_classes import ( + FIELD_TYPE_MAPPING, + KNOWN_DATA_PLATFORM_MAPPING, + App, + AppKey, + BoxType, + Chart, + QlikDataset, + QlikTable, + SchemaField as QlikDatasetSchemaField, + Sheet, + Space, + SpaceKey, +) +from datahub.ingestion.source.qlik_sense.qlik_api import QlikAPI +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalHandler, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionSourceBase, +) +from datahub.metadata.com.linkedin.pegasus2avro.common import ( + Status, + SubTypes, + TimeStamp, +) +from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( + DatasetLineageType, + DatasetProperties, + FineGrainedLineage, + FineGrainedLineageDownstreamType, + FineGrainedLineageUpstreamType, + Upstream, + UpstreamLineage, +) +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + MySqlDDL, + NullType, + SchemaField, + SchemaMetadata, +) +from datahub.metadata.schema_classes import ( + AuditStampClass, + ChangeAuditStampsClass, + ChartInfoClass, + DashboardInfoClass, + DataPlatformInstanceClass, + OwnerClass, + OwnershipClass, + OwnershipTypeClass, + SchemaFieldDataTypeClass, +) +from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result + +# Logger instance +logger = logging.getLogger(__name__) + + +@platform_name("Qlik Sense") +@config_class(QlikSourceConfig) +@support_status(SupportStatus.INCUBATING) +@capability(SourceCapability.DESCRIPTIONS, "Enabled by default") +@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability( + SourceCapability.OWNERSHIP, + "Enabled by default, configured using `ingest_owner`", +) +class QlikSenseSource(StatefulIngestionSourceBase, TestableSource): + """ + This plugin extracts the following: + - Qlik Sense Spaces and Apps as Container. + - Qlik Datasets + - Sheets as dashboard and its charts + """ + + config: QlikSourceConfig + reporter: QlikSourceReport + platform: str = "qlik-sense" + + def __init__(self, config: QlikSourceConfig, ctx: PipelineContext): + super(QlikSenseSource, self).__init__(config, ctx) + self.config = config + self.reporter = QlikSourceReport() + try: + self.qlik_api = QlikAPI(self.config) + except Exception as e: + logger.warning(e) + exit( + 1 + ) # Exit pipeline as we are not able to connect to Qlik Client Service. + + @staticmethod + def test_connection(config_dict: dict) -> TestConnectionReport: + test_report = TestConnectionReport() + try: + QlikAPI(QlikSourceConfig.parse_obj_allow_extras(config_dict)) + test_report.basic_connectivity = CapabilityReport(capable=True) + except Exception as e: + test_report.basic_connectivity = CapabilityReport( + capable=False, failure_reason=str(e) + ) + return test_report + + @classmethod + def create(cls, config_dict, ctx): + config = QlikSourceConfig.parse_obj(config_dict) + return cls(config, ctx) + + def _gen_space_key(self, space_id: str) -> SpaceKey: + return SpaceKey( + space=space_id, + platform=self.platform, + instance=self.config.platform_instance, + ) + + def _gen_app_key(self, app_id: str) -> AppKey: + return AppKey( + app=app_id, + platform=self.platform, + instance=self.config.platform_instance, + ) + + def _get_audit_stamp(self, date: datetime, username: str) -> AuditStampClass: + return AuditStampClass( + time=int(date.timestamp() * 1000), + actor=builder.make_user_urn(username), + ) + + def _get_allowed_spaces(self) -> List[Space]: + all_spaces = self.qlik_api.get_spaces() + allowed_spaces = [ + space + for space in all_spaces + if self.config.space_pattern.allowed(space.name) + ] + logger.info(f"Number of spaces = {len(all_spaces)}") + self.reporter.report_number_of_spaces(len(all_spaces)) + logger.info(f"Number of allowed spaces = {len(allowed_spaces)}") + return allowed_spaces + + def _gen_space_workunit(self, space: Space) -> Iterable[MetadataWorkUnit]: + """ + Map Qlik space to Datahub container + """ + owner_username: Optional[str] = None + if space.ownerId: + owner_username = self.qlik_api.get_user_name(space.ownerId) + yield from gen_containers( + container_key=self._gen_space_key(space.id), + name=space.name, + description=space.description, + sub_types=[BIContainerSubTypes.QLIK_SPACE], + extra_properties={Constant.TYPE: str(space.type)}, + owner_urn=builder.make_user_urn(owner_username) + if self.config.ingest_owner and owner_username + else None, + external_url=f"https://{self.config.tenant_hostname}/catalog?space_filter={space.id}", + created=int(space.createdAt.timestamp() * 1000), + last_modified=int(space.updatedAt.timestamp() * 1000), + ) + + def _gen_entity_status_aspect(self, entity_urn: str) -> MetadataWorkUnit: + return MetadataChangeProposalWrapper( + entityUrn=entity_urn, aspect=Status(removed=False) + ).as_workunit() + + def _gen_entity_owner_aspect( + self, entity_urn: str, user_name: str + ) -> MetadataWorkUnit: + aspect = OwnershipClass( + owners=[ + OwnerClass( + owner=builder.make_user_urn(user_name), + type=OwnershipTypeClass.DATAOWNER, + ) + ] + ) + return MetadataChangeProposalWrapper( + entityUrn=entity_urn, + aspect=aspect, + ).as_workunit() + + def _gen_dashboard_urn(self, dashboard_identifier: str) -> str: + return builder.make_dashboard_urn( + platform=self.platform, + platform_instance=self.config.platform_instance, + name=dashboard_identifier, + ) + + def _gen_dashboard_info_workunit( + self, sheet: Sheet, app_id: str + ) -> MetadataWorkUnit: + dashboard_urn = self._gen_dashboard_urn(sheet.id) + custom_properties: Dict[str, str] = {"chartCount": str(len(sheet.charts))} + dashboard_info_cls = DashboardInfoClass( + title=sheet.title, + description=sheet.description, + charts=[ + builder.make_chart_urn( + platform=self.platform, + platform_instance=self.config.platform_instance, + name=chart.qId, + ) + for chart in sheet.charts + ], + lastModified=ChangeAuditStampsClass( + created=self._get_audit_stamp(sheet.createdAt, sheet.ownerId), + lastModified=self._get_audit_stamp(sheet.updatedAt, sheet.ownerId), + ), + customProperties=custom_properties, + dashboardUrl=f"https://{self.config.tenant_hostname}/sense/app/{app_id}/sheet/{sheet.id}/state/analysis", + ) + return MetadataChangeProposalWrapper( + entityUrn=dashboard_urn, aspect=dashboard_info_cls + ).as_workunit() + + def _gen_charts_workunit( + self, charts: List[Chart], input_tables: List[QlikTable], app_id: str + ) -> Iterable[MetadataWorkUnit]: + """ + Map Qlik Chart to Datahub Chart + """ + input_tables_urns: List[str] = [] + for table in input_tables: + table_identifier = self._get_app_table_identifier(table) + if table_identifier: + input_tables_urns.append(self._gen_qlik_dataset_urn(table_identifier)) + + for chart in charts: + chart_urn = builder.make_chart_urn( + platform=self.platform, + platform_instance=self.config.platform_instance, + name=chart.qId, + ) + + yield self._gen_entity_status_aspect(chart_urn) + + custom_properties = { + "Dimension": str(chart.qDimension), + "Measure": str(chart.qMeasure), + } + + yield MetadataChangeProposalWrapper( + entityUrn=chart_urn, + aspect=ChartInfoClass( + title=chart.title, + description=chart.visualization, + lastModified=ChangeAuditStampsClass(), + customProperties=custom_properties, + inputs=input_tables_urns, + ), + ).as_workunit() + + yield from add_entity_to_container( + container_key=self._gen_app_key(app_id), + entity_type="chart", + entity_urn=chart_urn, + ) + + def _gen_sheets_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: + """ + Map Qlik Sheet to Datahub dashboard + """ + for sheet in app.sheets: + dashboard_urn = self._gen_dashboard_urn(sheet.id) + + yield self._gen_entity_status_aspect(dashboard_urn) + + yield self._gen_dashboard_info_workunit(sheet, app.id) + + yield from add_entity_to_container( + container_key=self._gen_app_key(app.id), + entity_type="dashboard", + entity_urn=dashboard_urn, + ) + + dpi_aspect = self._gen_dataplatform_instance_aspect(dashboard_urn) + if dpi_aspect: + yield dpi_aspect + + owner_username = self.qlik_api.get_user_name(sheet.ownerId) + if self.config.ingest_owner and owner_username: + yield self._gen_entity_owner_aspect(dashboard_urn, owner_username) + + yield from self._gen_charts_workunit(sheet.charts, app.tables, app.id) + + def _gen_app_table_upstream_lineage( + self, dataset_urn: str, table: QlikTable + ) -> Optional[MetadataWorkUnit]: + upstream_dataset_urn: Optional[str] = None + if table.type == BoxType.BLACKBOX: + upstream_dataset_platform_detail = ( + self.config.data_connection_to_platform_instance.get( + table.dataconnectorName, + PlatformDetail( + env=self.config.env, + platform_instance=self.config.platform_instance, + ), + ) + ) + if not table.selectStatement: + return None + upstream_dataset_urn = create_lineage_sql_parsed_result( + query=table.selectStatement.strip(), + default_db=None, + platform=KNOWN_DATA_PLATFORM_MAPPING.get( + table.dataconnectorPlatform, table.dataconnectorPlatform + ), + env=upstream_dataset_platform_detail.env, + platform_instance=upstream_dataset_platform_detail.platform_instance, + ).in_tables[0] + elif table.type == BoxType.LOADFILE: + upstream_dataset_urn = self._gen_qlik_dataset_urn( + f"{table.spaceId}.{table.databaseName}".lower() + ) + + if upstream_dataset_urn: + # Generate finegrained lineage + fine_grained_lineages = [ + FineGrainedLineage( + upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, + upstreams=[ + builder.make_schema_field_urn(upstream_dataset_urn, field.name) + ], + downstreamType=FineGrainedLineageDownstreamType.FIELD, + downstreams=[ + builder.make_schema_field_urn(dataset_urn, field.name) + ], + ) + for field in table.datasetSchema + ] + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=UpstreamLineage( + upstreams=[ + Upstream( + dataset=upstream_dataset_urn, type=DatasetLineageType.COPY + ) + ], + fineGrainedLineages=fine_grained_lineages, + ), + ).as_workunit() + else: + return None + + def _gen_app_table_properties( + self, dataset_urn: str, table: QlikTable + ) -> MetadataWorkUnit: + dataset_properties = DatasetProperties( + name=table.tableName, + qualifiedName=table.tableAlias, + ) + dataset_properties.customProperties.update( + { + Constant.TYPE: "Qlik Table", + Constant.DATACONNECTORID: table.dataconnectorid, + Constant.DATACONNECTORNAME: table.dataconnectorName, + } + ) + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=dataset_properties + ).as_workunit() + + def _get_app_table_identifier(self, table: QlikTable) -> Optional[str]: + if table.tableQri: + return table.tableQri.lower() + return None + + def _gen_app_tables_workunit( + self, tables: List[QlikTable], app_id: str + ) -> Iterable[MetadataWorkUnit]: + for table in tables: + table_identifier = self._get_app_table_identifier(table) + if not table_identifier: + continue + dataset_urn = self._gen_qlik_dataset_urn(table_identifier) + + yield self._gen_entity_status_aspect(dataset_urn) + + yield self._gen_schema_metadata(table_identifier, table.datasetSchema) + + yield self._gen_app_table_properties(dataset_urn, table) + + yield from add_entity_to_container( + container_key=self._gen_app_key(app_id), + entity_type="dataset", + entity_urn=dataset_urn, + ) + + dpi_aspect = self._gen_dataplatform_instance_aspect(dataset_urn) + if dpi_aspect: + yield dpi_aspect + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=SubTypes(typeNames=[DatasetSubTypes.QLIK_DATASET]), + ).as_workunit() + + upstream_lineage_workunit = self._gen_app_table_upstream_lineage( + dataset_urn, table + ) + if upstream_lineage_workunit: + yield upstream_lineage_workunit + + def _gen_app_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: + """ + Map Qlik App to Datahub container + """ + owner_username = self.qlik_api.get_user_name(app.ownerId) + yield from gen_containers( + container_key=self._gen_app_key(app.id), + name=app.qTitle, + description=app.description, + sub_types=[BIContainerSubTypes.QLIK_APP], + parent_container_key=self._gen_space_key(app.spaceId), + extra_properties={Constant.QRI: app.qri, Constant.USAGE: app.qUsage}, + owner_urn=builder.make_user_urn(owner_username) + if self.config.ingest_owner and owner_username + else None, + external_url=f"https://{self.config.tenant_hostname}/sense/app/{app.id}/overview", + created=int(app.createdAt.timestamp() * 1000), + last_modified=int(app.updatedAt.timestamp() * 1000), + ) + yield from self._gen_sheets_workunit(app) + yield from self._gen_app_tables_workunit(app.tables, app.id) + + def _gen_qlik_dataset_urn(self, dataset_identifier: str) -> str: + return builder.make_dataset_urn_with_platform_instance( + name=dataset_identifier, + env=self.config.env, + platform=self.platform, + platform_instance=self.config.platform_instance, + ) + + def _gen_dataplatform_instance_aspect( + self, entity_urn: str + ) -> Optional[MetadataWorkUnit]: + if self.config.platform_instance: + aspect = DataPlatformInstanceClass( + platform=builder.make_data_platform_urn(self.platform), + instance=builder.make_dataplatform_instance_urn( + self.platform, self.config.platform_instance + ), + ) + return MetadataChangeProposalWrapper( + entityUrn=entity_urn, aspect=aspect + ).as_workunit() + else: + return None + + def _gen_schema_fields( + self, schema: List[QlikDatasetSchemaField] + ) -> List[SchemaField]: + schema_fields: List[SchemaField] = [] + for field in schema: + schema_field = SchemaField( + fieldPath=field.name, + type=SchemaFieldDataTypeClass( + type=FIELD_TYPE_MAPPING.get(field.dataType, NullType)() + if field.dataType + else NullType() + ), + nativeDataType=field.dataType if field.dataType else "", + nullable=field.nullable, + isPartOfKey=field.primaryKey, + ) + schema_fields.append(schema_field) + return schema_fields + + def _gen_schema_metadata( + self, dataset_identifier: str, dataset_schema: List[QlikDatasetSchemaField] + ) -> MetadataWorkUnit: + dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) + + schema_metadata = SchemaMetadata( + schemaName=dataset_identifier, + platform=builder.make_data_platform_urn(self.platform), + version=0, + hash="", + platformSchema=MySqlDDL(tableSchema=""), + fields=self._gen_schema_fields(dataset_schema), + ) + + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=schema_metadata + ).as_workunit() + + def _gen_dataset_properties( + self, dataset_urn: str, dataset: QlikDataset + ) -> MetadataWorkUnit: + dataset_properties = DatasetProperties( + name=dataset.name, + description=dataset.description, + qualifiedName=dataset.name, + externalUrl=f"https://{self.config.tenant_hostname}/dataset/{dataset.itemId}", + created=TimeStamp(time=int(dataset.createdAt.timestamp() * 1000)), + lastModified=TimeStamp(time=int(dataset.updatedAt.timestamp() * 1000)), + ) + dataset_properties.customProperties.update( + { + Constant.QRI: dataset.secureQri, + Constant.SPACEID: dataset.spaceId, + Constant.TYPE: "Qlik Dataset", + Constant.DATASETTYPE: dataset.type, + Constant.SIZE: str(dataset.size), + Constant.ROWCOUNT: str(dataset.rowCount), + } + ) + + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=dataset_properties + ).as_workunit() + + def _get_qlik_dataset_identifier(self, dataset: QlikDataset) -> str: + return f"{dataset.spaceId}.{dataset.name}".lower() + + def _gen_dataset_workunit(self, dataset: QlikDataset) -> Iterable[MetadataWorkUnit]: + dataset_identifier = self._get_qlik_dataset_identifier(dataset) + dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) + + yield self._gen_entity_status_aspect(dataset_urn) + + yield self._gen_schema_metadata(dataset_identifier, dataset.datasetSchema) + + yield self._gen_dataset_properties(dataset_urn, dataset) + + yield from add_entity_to_container( + container_key=self._gen_space_key(dataset.spaceId), + entity_type="dataset", + entity_urn=dataset_urn, + ) + + dpi_aspect = self._gen_dataplatform_instance_aspect(dataset_urn) + if dpi_aspect: + yield dpi_aspect + + owner_username = self.qlik_api.get_user_name(dataset.ownerId) + if self.config.ingest_owner and owner_username: + yield self._gen_entity_owner_aspect(dataset_urn, owner_username) + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=SubTypes(typeNames=[DatasetSubTypes.QLIK_DATASET]), + ).as_workunit() + + def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: + return [ + *super().get_workunit_processors(), + StaleEntityRemovalHandler.create( + self, self.config, self.ctx + ).workunit_processor, + ] + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + """ + Datahub Ingestion framework invoke this method + """ + logger.info("Qlik Sense plugin execution is started") + for space in self._get_allowed_spaces(): + yield from self._gen_space_workunit(space) + for item in self.qlik_api.get_items(): + if isinstance(item, App): + yield from self._gen_app_workunit(item) + elif isinstance(item, QlikDataset): + yield from self._gen_dataset_workunit(item) + + def get_report(self) -> SourceReport: + return self.reporter diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/websocket_connection.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/websocket_connection.py new file mode 100644 index 0000000000000..01ca9415f886a --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/websocket_connection.py @@ -0,0 +1,55 @@ +import json +from typing import Dict, List, Optional, Union + +from websocket import WebSocket, create_connection + +from datahub.ingestion.source.qlik_sense.config import Constant + + +class WebsocketConnection: + def __init__(self, tenant_hostname: str, api_key: str, app_id: str) -> None: + self.websocket_url = f"wss://{tenant_hostname}/app" + self.socket_connection: Optional[WebSocket] = create_connection( + f"{self.websocket_url}/{app_id}", + header={"Authorization": f"Bearer {api_key}"}, + ) + self.request_id = 0 + self.handle = [-1] + + def _build_websocket_request_dict( + self, method: str, params: Union[Dict, List] = {} + ) -> Dict: + return { + "jsonrpc": "2.0", + "id": self.request_id, + "handle": self.handle[-1], + "method": method, + "params": params, + } + + def _send_request(self, request: Dict) -> Dict: + if self.socket_connection: + self.socket_connection.send(json.dumps(request)) + resp = self.socket_connection.recv() + if request["handle"] == -1: + resp = self.socket_connection.recv() + return json.loads(resp)[Constant.RESULT] + return {} + + def websocket_send_request( + self, method: str, params: Union[Dict, List] = {} + ) -> Dict: + """ + Method to send request to websocket + """ + self.request_id += 1 + request = self._build_websocket_request_dict(method, params) + response = self._send_request(request=request) + handle = response.get(Constant.QRETURN, {}).get(Constant.QHANDLE) + if handle: + self.handle.append(handle) + return response + + def close_websocket(self) -> None: + if self.socket_connection: + self.socket_connection.close() diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json new file mode 100644 index 0000000000000..6917b833d05a3 --- /dev/null +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json @@ -0,0 +1,1680 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-sense", + "instance": "qlik_sense_platform", + "space": "659d0e41d1b0ecce6eebc9b1", + "type": "SpaceType.SHARED" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=659d0e41d1b0ecce6eebc9b1", + "name": "test_space", + "description": "", + "created": { + "time": 1704771818002 + }, + "lastModified": { + "time": 1704771818002 + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546102, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546103, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546104, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546105, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546106, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546106, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-sense", + "instance": "qlik_sense_platform", + "space": "personal-space-id", + "type": "SpaceType.PERSONAL" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=personal-space-id", + "name": "personal_space", + "description": "", + "created": { + "time": 1707327546001 + }, + "lastModified": { + "time": 1707327546001 + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546121, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546122, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546123, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546124, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546124, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-sense", + "instance": "qlik_sense_platform", + "app": "f0714ca7-7093-49e4-8b58-47bb38563647", + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647", + "usage": "ANALYTICS" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/overview", + "name": "IPL_Matches_2022", + "description": "", + "created": { + "time": 1704803736545 + }, + "lastModified": { + "time": 1705297020333 + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546126, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546127, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546127, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik App" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546128, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546129, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + } + }, + "systemMetadata": { + "lastObserved": 1707327546130, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546130, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546132, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "chartCount": "1" + }, + "title": "New ds sheet", + "description": "", + "charts": [ + "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1705296709704, + "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" + }, + "lastModified": { + "time": 1706511226868, + "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" + } + }, + "dashboardUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/sheet/f4f57386-263a-4ec9-b40c-abcd2467f423/state/analysis" + } + }, + "systemMetadata": { + "lastObserved": 1707327546132, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + }, + "systemMetadata": { + "lastObserved": 1707327546133, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546134, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546135, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546136, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546137, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "Dimension": "[AxisProperty(Title='City', Min='NaN', Max='NaN')]", + "Measure": "[AxisProperty(Title='Sum(Date)', Min='89411', Max='2144260')]" + }, + "title": "Test_chart", + "description": "scatterplot", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707803447230, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + }, + "systemMetadata": { + "lastObserved": 1707393018679, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018680, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546139, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546140, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "type": "Qlik Table", + "dataconnectorid": "bb5be407-d3d3-4f19-858c-e71d593f09ae", + "dataconnectorName": "Google_BigQuery_harshal-playground-306419" + }, + "name": "test_table", + "qualifiedName": "test_table", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327546142, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + }, + "systemMetadata": { + "lastObserved": 1707327546142, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546143, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546144, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,google-cloud.harshal-playground-306419.test_dataset.test_table,DEV)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,google-cloud.harshal-playground-306419.test_dataset.test_table,DEV),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD),name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018687, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546145, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546147, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546147, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "type": "Qlik Table", + "dataconnectorid": "87d7bc7e-77d8-40dc-a251-3a35ec107b4e", + "dataconnectorName": "DataFiles" + }, + "name": "IPL_Matches_2022", + "qualifiedName": "IPL_Matches_2022", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327546149, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + }, + "systemMetadata": { + "lastObserved": 1707327546150, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546151, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546151, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD),City)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD),Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD),Date)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018696, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546153, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546155, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Season", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546155, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "type": "Qlik Dataset", + "datasetType": "DELIMETED", + "size": "38168", + "rowCount": "74" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/659d8aef12794f37026cb262", + "name": "IPL_Matches_2022.csv", + "qualifiedName": "IPL_Matches_2022.csv", + "description": "", + "created": { + "time": 1704803735527 + }, + "lastModified": { + "time": 1704804512453 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327546158, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + } + }, + "systemMetadata": { + "lastObserved": 1707327546159, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546160, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546161, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546161, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546162, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546163, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "personal-space-id.test_tabl", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546164, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "qri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", + "spaceId": "personal-space-id", + "type": "Qlik Dataset", + "datasetType": "CONNECTION_BASED_DATASET", + "size": "0", + "rowCount": "1" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/65a137c8d5a03b02d359624a", + "name": "test_tabl", + "qualifiedName": "test_tabl", + "description": "", + "created": { + "time": 1705044592288 + }, + "lastModified": { + "time": 1705045296325 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327546166, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" + } + }, + "systemMetadata": { + "lastObserved": 1707327546167, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546167, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546168, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546169, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546170, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546170, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546171, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546172, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546172, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546173, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546174, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546175, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546176, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json new file mode 100644 index 0000000000000..5b47c09a79143 --- /dev/null +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json @@ -0,0 +1,1517 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-sense", + "space": "659d0e41d1b0ecce6eebc9b1", + "type": "SpaceType.SHARED" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=659d0e41d1b0ecce6eebc9b1", + "name": "test_space", + "description": "", + "created": { + "time": 1704771818002 + }, + "lastModified": { + "time": 1704771818002 + } + } + }, + "systemMetadata": { + "lastObserved": 1707327442983, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327442985, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense" + } + }, + "systemMetadata": { + "lastObserved": 1707327442985, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327442986, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327442987, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327442988, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-sense", + "space": "personal-space-id", + "type": "SpaceType.PERSONAL" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=personal-space-id", + "name": "personal_space", + "description": "", + "created": { + "time": 1707327442833 + }, + "lastModified": { + "time": 1707327442833 + } + } + }, + "systemMetadata": { + "lastObserved": 1707327443007, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443008, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense" + } + }, + "systemMetadata": { + "lastObserved": 1707327443010, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443010, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443011, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-sense", + "app": "f0714ca7-7093-49e4-8b58-47bb38563647", + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647", + "usage": "ANALYTICS" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/overview", + "name": "IPL_Matches_2022", + "description": "", + "created": { + "time": 1704803736545 + }, + "lastModified": { + "time": 1705297020333 + } + } + }, + "systemMetadata": { + "lastObserved": 1707327443012, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443014, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense" + } + }, + "systemMetadata": { + "lastObserved": 1707327443014, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik App" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443015, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327443016, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + } + }, + "systemMetadata": { + "lastObserved": 1707327443017, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443018, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443019, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "chartCount": "1" + }, + "title": "New ds sheet", + "description": "", + "charts": [ + "urn:li:chart:(qlik-sense,QYUUb)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1705296709704, + "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" + }, + "lastModified": { + "time": 1706511226868, + "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" + } + }, + "dashboardUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/sheet/f4f57386-263a-4ec9-b40c-abcd2467f423/state/analysis" + } + }, + "systemMetadata": { + "lastObserved": 1707327443020, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + }, + "systemMetadata": { + "lastObserved": 1707327443022, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327443023, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443024, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,QYUUb)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443025, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,QYUUb)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "Dimension": "[AxisProperty(Title='City', Min='NaN', Max='NaN')]", + "Measure": "[AxisProperty(Title='Sum(Date)', Min='89411', Max='2144260')]" + }, + "title": "Test_chart", + "description": "scatterplot", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707803447013, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,QYUUb)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + }, + "systemMetadata": { + "lastObserved": 1707393018435, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,QYUUb)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018436, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443028, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443029, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "type": "Qlik Table", + "dataconnectorid": "bb5be407-d3d3-4f19-858c-e71d593f09ae", + "dataconnectorName": "Google_BigQuery_harshal-playground-306419" + }, + "name": "test_table", + "qualifiedName": "test_table", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443031, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + }, + "systemMetadata": { + "lastObserved": 1707327443032, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443033, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_dataset.test_table,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_dataset.test_table,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD),name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018442, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443035, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443037, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443038, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "type": "Qlik Table", + "dataconnectorid": "87d7bc7e-77d8-40dc-a251-3a35ec107b4e", + "dataconnectorName": "DataFiles" + }, + "name": "IPL_Matches_2022", + "qualifiedName": "IPL_Matches_2022", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443040, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + }, + "systemMetadata": { + "lastObserved": 1707327443041, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443042, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD),City)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD),Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD),Date)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018451, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443045, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443048, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Season", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443050, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "type": "Qlik Dataset", + "datasetType": "DELIMETED", + "size": "38168", + "rowCount": "74" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/659d8aef12794f37026cb262", + "name": "IPL_Matches_2022.csv", + "qualifiedName": "IPL_Matches_2022.csv", + "description": "", + "created": { + "time": 1704803735527 + }, + "lastModified": { + "time": 1704804512453 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443055, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + } + }, + "systemMetadata": { + "lastObserved": 1707327443057, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327443058, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443060, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443062, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443065, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "personal-space-id.test_tabl", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443066, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "qri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", + "spaceId": "personal-space-id", + "type": "Qlik Dataset", + "datasetType": "CONNECTION_BASED_DATASET", + "size": "0", + "rowCount": "1" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/65a137c8d5a03b02d359624a", + "name": "test_tabl", + "qualifiedName": "test_tabl", + "description": "", + "created": { + "time": 1705044592288 + }, + "lastModified": { + "time": 1705045296325 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443069, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + } + }, + "systemMetadata": { + "lastObserved": 1707327443071, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327443073, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443073, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443074, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443076, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443076, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443078, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443079, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443080, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443081, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443082, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443082, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py new file mode 100644 index 0000000000000..00a50b7dbe54c --- /dev/null +++ b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py @@ -0,0 +1,1088 @@ +from typing import Any, Dict +from unittest.mock import patch + +import pytest + +from datahub.ingestion.run.pipeline import Pipeline +from tests.test_helpers import mce_helpers + + +def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: + api_vs_response: Dict[str, Dict] = { + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/api-keys": { + "method": "GET", + "status_code": 200, + "json": {}, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/spaces": { + "method": "GET", + "status_code": 200, + "json": { + "data": [ + { + "id": "659d0e41d1b0ecce6eebc9b1", + "type": "shared", + "ownerId": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "name": "test_space", + "description": "", + "meta": { + "actions": ["create", "delete", "read", "update"], + "roles": [], + "assignableRoles": [ + "codeveloper", + "consumer", + "dataconsumer", + "facilitator", + "producer", + ], + }, + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/spaces/659d0e41d1b0ecce6eebc9b1" + }, + "assignments": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/spaces/659d0e41d1b0ecce6eebc9b1/assignments" + }, + }, + "createdAt": "2024-01-09T09:13:38.002Z", + "createdBy": "657b5abe656297cec3d8b205", + "updatedAt": "2024-01-09T09:13:38.002Z", + } + ] + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items": { + "method": "GET", + "status_code": 200, + "json": { + "data": [ + { + "name": "test_app", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "thumbnailId": "", + "resourceAttributes": { + "_resourcetype": "app", + "createdDate": "2024-01-25T17:52:24.922Z", + "description": "", + "dynamicColor": "", + "hasSectionAccess": False, + "id": "f0714ca7-7093-49e4-8b58-47bb38563647", + "isDirectQueryMode": False, + "lastReloadTime": "2024-01-25T17:56:00.902Z", + "modifiedDate": "2024-01-25T17:56:02.045Z", + "name": "test_app", + "originAppId": "", + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "publishTime": "", + "published": False, + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "thumbnail": "", + "usage": "ANALYTICS", + }, + "resourceCustomAttributes": None, + "resourceUpdatedAt": "2024-01-25T17:56:02Z", + "resourceType": "app", + "resourceId": "f0714ca7-7093-49e4-8b58-47bb38563647", + "resourceCreatedAt": "2024-01-25T17:52:24Z", + "id": "65b29fd9435c545ed042a807", + "createdAt": "2024-01-25T17:52:25Z", + "updatedAt": "2024-01-25T17:56:02Z", + "creatorId": "657b5abe656297cec3d8b205", + "updaterId": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "isFavorited": False, + "collectionIds": [], + "meta": { + "isFavorited": False, + "tags": [ + {"id": "659ce561640a2affcf0d629f", "name": "test_tag"} + ], + "collections": [], + }, + "ownerId": "657b5abe656297cec3d8b205", + "resourceReloadEndTime": "2024-01-25T17:56:00Z", + "resourceReloadStatus": "ok", + "resourceSize": {"appFile": 2057, "appMemory": 78}, + "itemViews": {"total": 3, "trend": 0.3, "unique": 1}, + }, + { + "name": "IPL_Matches_2022.csv", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "resourceAttributes": { + "appType": "QIX-DF", + "dataStoreName": "DataFilesStore", + "dataStoreType": "qix-datafiles", + "qri": "qdf:qix-datafiles:ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A:sid@659d0e41d1b0ecce6eebc9b1:IPL_Matches_2022.csv", + "secureQri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "sourceSystemId": "QIX-DF_6fb35d21-24d0-474b-bf8b-536c6e9dc717", + "technicalDescription": "", + "technicalName": "IPL_Matches_2022.csv", + "type": "DELIMETED", + "version": "2", + }, + "resourceCustomAttributes": None, + "resourceUpdatedAt": "2024-01-09T18:18:32Z", + "resourceType": "dataset", + "resourceSubType": "qix-df", + "resourceId": "659d8aef92df266ef3aa5a7c", + "resourceCreatedAt": "2024-01-09T18:05:35Z", + "id": "659d8aef12794f37026cb262", + "createdAt": "2024-01-09T18:05:35Z", + "updatedAt": "2024-01-09T18:18:32Z", + "creatorId": "657b5abe656297cec3d8b205", + "updaterId": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "isFavorited": False, + "collectionIds": [], + "meta": { + "tags": [], + "collections": [], + }, + "ownerId": "657b5abe656297cec3d8b205", + "resourceReloadEndTime": "", + "resourceReloadStatus": "", + "resourceSize": {"appFile": 0, "appMemory": 0}, + "itemViews": {}, + }, + { + "name": "test_tabl", + "resourceAttributes": { + "appType": "CONNECTION_BASED_DATASET", + "dataStoreName": "gbqqri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#QgUSMctzlGjr47d2tMP35YlS93h78jVECRqtByqxPEE", + "dataStoreType": "gbq", + "qri": "a4d1b9ef-e629-4943-809a-2713aa3a5345", + "secureQri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", + "sourceSystemId": "", + "technicalDescription": "", + "technicalName": "harshal-playground-306419'.'test_dataset'.'test_table", + "type": "CONNECTION_BASED_DATASET", + "version": "2", + }, + "resourceCustomAttributes": None, + "resourceUpdatedAt": "2024-01-12T13:11:36Z", + "resourceType": "dataset", + "resourceSubType": "connection_based_dataset", + "resourceId": "65a137c849f82a37c625151b", + "resourceCreatedAt": "2024-01-12T12:59:52Z", + "id": "65a137c8d5a03b02d359624a", + "createdAt": "2024-01-12T12:59:52Z", + "updatedAt": "2024-01-12T13:11:36Z", + "creatorId": "657b5abe656297cec3d8b205", + "updaterId": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "isFavorited": False, + "collectionIds": [], + "meta": { + "tags": [], + "collections": [], + }, + "ownerId": "657b5abe656297cec3d8b205", + "resourceReloadEndTime": "", + "resourceReloadStatus": "", + "resourceSize": {"appFile": 0, "appMemory": 0}, + }, + ], + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items" + } + }, + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/users/657b5abe656297cec3d8b205": { + "method": "GET", + "status_code": 200, + "json": { + "id": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "status": "active", + "subject": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "name": "Shubham jagtap", + "email": "Shubham.Jagtap@gslab.com", + "roles": [ + "TenantAdmin", + "AnalyticsAdmin", + "AuditAdmin", + "DataAdmin", + "Developer", + "SharedSpaceCreator", + "PrivateAnalyticsContentCreator", + "DataServicesContributor", + ], + "groups": [], + "createdAt": "2023-12-14T19:42:54.417Z", + "lastUpdatedAt": "2024-01-25T06:28:22.629Z", + "created": "2023-12-14T19:42:54.417Z", + "lastUpdated": "2024-01-25T06:28:22.629Z", + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/users/657b5abe656297cec3d8b205" + } + }, + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/data-sets/659d8aef92df266ef3aa5a7c": { + "method": "GET", + "status_code": 200, + "json": { + "id": "659d8aef92df266ef3aa5a7c", + "name": "IPL_Matches_2022.csv", + "description": "", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "ownerId": "657b5abe656297cec3d8b205", + "createdTime": "2024-01-09T18:05:35.527Z", + "createdBy": "657b5abe656297cec3d8b205", + "lastModifiedTime": "2024-01-09T18:18:32.453Z", + "lastModifiedBy": "657b5abe656297cec3d8b205", + "version": 2, + "technicalDescription": "", + "technicalName": "IPL_Matches_2022.csv", + "properties": {"ModifiedByProfileService": False}, + "dataAssetInfo": { + "id": "659d8aef92df266ef3aa5a7b", + "name": "test_space", + "dataStoreInfo": { + "id": "659d8aef92df266ef3aa5a7a", + "name": "DataFilesStore", + "type": "qix-datafiles", + }, + }, + "operational": { + "size": 38168, + "rowCount": 74, + "lastLoadTime": "2024-01-09T18:05:35.527Z", + "logMessage": '{\n "cloudEventsVersion": "0.1",\n "source": "com.qlik/qix-datafiles",\n "contentType": "application/json",\n "eventID": "48c935d3-499c-4333-96e6-c8922af6e238",\n "eventType": "com.qlik.datafile.created",\n "eventTypeVersion": "0.0.1",\n "eventTime": "2024-01-09T18:05:35.834061Z",\n "extensions": {\n "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A",\n "userId": "657b5abe656297cec3d8b205",\n "header": {\n "traceparent": [\n "00-000000000000000063d931098d8f607c-7f2bea16c07c1c3a-01"\n ]\n }\n },\n "data": {\n "id": "6fb35d21-24d0-474b-bf8b-536c6e9dc717",\n "name": "IPL_Matches_2022.csv",\n "createdDate": "2024-01-09T18:05:35.5279392Z",\n "modifiedDate": "2024-01-09T18:05:35.828813Z",\n "createdByUser": "657b5abe656297cec3d8b205",\n "modifiedByUser": "657b5abe656297cec3d8b205",\n "ownerId": "657b5abe656297cec3d8b205",\n "spaceId": "659d0e41d1b0ecce6eebc9b1",\n "size": 38168,\n "contentUpdated": true,\n "isInternal": false,\n "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4"\n },\n "messageTopic": "system-events.datafiles/Created/ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A/6fb35d21-24d0-474b-bf8b-536c6e9dc717"\n}', + "status": "com.qlik.datafile.created", + "location": "48c935d3-499c-4333-96e6-c8922af6e238", + "contentUpdated": True, + "lastUpdateTime": "2024-01-09T18:05:35.828Z", + }, + "schema": { + "dataFields": [ + { + "name": "ID", + "index": 0, + "dataType": { + "type": "INTEGER", + "properties": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$integer", "$numeric"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + }, + { + "name": "City", + "index": 1, + "dataType": { + "type": "STRING", + "properties": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$text", "$ascii"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + }, + { + "name": "Date", + "index": 2, + "dataType": { + "type": "DATE", + "properties": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$integer", "$numeric", "$date", "$timestamp"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + }, + { + "name": "Season", + "index": 3, + "dataType": { + "type": "INTEGER", + "properties": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$integer", "$numeric"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + }, + ], + "loadOptions": { + "qDataFormat": { + "qType": "CSV", + "qLabel": "embedded labels", + "qQuote": "msq", + "qDelimiter": { + "qName": "Comma", + "qScriptCode": "','", + "qNumber": 44, + }, + "qCodePage": 28591, + "qHeaderSize": 0, + "qRecordSize": 0, + "qTabSize": 0, + } + }, + "effectiveDate": "2024-01-09T18:05:39.713Z", + "overrideSchemaAnomalies": False, + }, + "qri": "qdf:qix-datafiles:ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A:sid@659d0e41d1b0ecce6eebc9b1:IPL_Matches_2022.csv", + "secureQri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "type": "DELIMETED", + "classifications": { + "personalInformation": [], + "sensitiveInformation": [], + }, + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/data-sets/65a137c849f82a37c625151b": { + "method": "GET", + "status_code": 200, + "json": { + "id": "65a137c849f82a37c625151b", + "name": "test_tabl", + "description": "", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "ownerId": "657b5abe656297cec3d8b205", + "createdTime": "2024-01-12T12:59:52.288Z", + "createdBy": "657b5abe656297cec3d8b205", + "lastModifiedTime": "2024-01-12T13:11:36.325Z", + "lastModifiedBy": "657b5abe656297cec3d8b205", + "version": 2, + "technicalDescription": "", + "technicalName": "harshal-playground-306419'.'test_dataset'.'test_table", + "properties": {"ModifiedByProfileService": False}, + "dataAssetInfo": { + "id": "65a137c849f82a37c625151a", + "name": "gbqqri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#QgUSMctzlGjr47d2tMP35YlS93h78jVECRqtByqxPEE", + "dataStoreInfo": { + "id": "65a137c849f82a37c6251519", + "name": "gbqqri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#QgUSMctzlGjr47d2tMP35YlS93h78jVECRqtByqxPEE", + "type": "gbq", + }, + }, + "operational": { + "rowCount": 1, + "contentUpdated": False, + "tableConnectionInfo": { + "tableName": "test_table", + "selectionScript": "[test_table]:\nSELECT name\nFROM `harshal-playground-306419`.`test_dataset`.`test_table`;", + "additionalProperties": { + "fields": '[{"name":"name","fullName":"name","nativeType":"STRING","nativeFieldInfo":{"dataType":12,"name":"name","nullable":1,"ordinalPostion":1,"scale":0,"size":65535,"typeName":"STRING"},"customProperties":[],"isSelected":true}]', + "tableRequestParameters": '[{\\"name\\":\\"database\\",\\"value\\":\\"harshal-playground-306419\\"},{\\"name\\":\\"owner\\",\\"value\\":\\"test_dataset\\"}]', + }, + }, + }, + "schema": { + "dataFields": [ + { + "name": "name", + "index": 0, + "dataType": { + "type": "STRING", + "properties": {"qType": "A", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$text", "$ascii"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + } + ], + "loadOptions": {}, + "effectiveDate": "2024-01-12T12:59:54.522Z", + "anomalies": ["$warning-single-text-column"], + "overrideSchemaAnomalies": False, + }, + "qri": "a4d1b9ef-e629-4943-809a-2713aa3a5345", + "secureQri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", + "type": "CONNECTION_BASED_DATASET", + "classifications": { + "personalInformation": [], + "sensitiveInformation": [], + }, + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/actions/expand?node=qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647&level=TABLE": { + "method": "GET", + "status_code": 200, + "json": { + "graph": { + "id": "", + "directed": True, + "type": "TABLE", + "label": "Expansion for qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647", + "nodes": { + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc": { + "label": "IPL_Matches_2022", + "metadata": {"fields": 1, "type": "TABLE"}, + }, + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE": { + "label": "test_table", + "metadata": {"fields": 1, "type": "TABLE"}, + }, + }, + "edges": [ + { + "relation": "from", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#t91dFUED_ebvKHOjauxSOOCnlRZQTwEkNHv_bjUl7AY#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + }, + ], + "metadata": {}, + } + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/actions/expand?node=qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647%23FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc&level=FIELD": { + "method": "GET", + "status_code": 200, + "json": { + "graph": { + "id": "", + "directed": True, + "type": "FIELD", + "label": "Expansion for qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + "nodes": { + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#-NAdo6895jVLliOXp13Do7e1kDNDDPwxZ0CDuGFJb8A": { + "label": "City", + "metadata": {"type": "FIELD"}, + }, + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#3DAdoo7e1kDNDDPwxZ0CD3Do7e1kDNDDPwxZ0CDuGFJ": { + "label": "Date", + "metadata": {"type": "FIELD"}, + }, + }, + "edges": [ + { + "relation": "from", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#fpA_oxHhnvRu-RrTC_INPo-TEykis1Y_6e-Qy0rKy2I#ti1Pyd82WD92n5Mafl23Gs1Pr_pwJ1tyFIWtaTJfslc#BcCb_mytkkVA-HSHbFLRYOHFY-O_55m8X_JG7DNzMNE", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#-NAdo6895jVLliOXp13Do7e1kDNDDPwxZ0CDuGFJb8A", + }, + { + "relation": "from", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#fpA_oxHhnvRu-RrTC_INPo-TEykis1Y_6e-Qy0rKy2I#ti1Pyd82WD92n5Mafl23Gs1Pr_pwJ1tyFIWtaTJfslc#BcCb_mytkkVA-HSHbFLRYOHFY-O_58X8X_JG7DNzMNE", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#3DAdoo7e1kDNDDPwxZ0CD3Do7e1kDNDDPwxZ0CDuGFJ", + }, + ], + "metadata": {}, + } + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/actions/expand?node=qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647%23Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE&level=FIELD": { + "method": "GET", + "status_code": 200, + "json": { + "graph": { + "id": "", + "directed": True, + "type": "FIELD", + "label": "Expansion for qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE", + "nodes": { + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok": { + "label": "name", + "metadata": {"type": "FIELD"}, + } + }, + "edges": [ + { + "relation": "read", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#JOKG8u7CvizvGXwrFsyXRU0yKr2rL2WFD5djpH9bj5Q#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok", + } + ], + "metadata": {}, + } + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/overview": { + "method": "POST", + "response_list": [ + { + "status_code": 200, + "json": { + "resources": [ + { + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#-NAdo6895jVLliOXp13Do7e1kDNDDPwxZ0CDuGFJb8A", + "lineage": [ + { + "resourceLabel": "d7a6c03238b0c25b3d5c8631e1121807.qvd", + "resourceQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#_s2Ws5RVxAArzAZlmghwjHNGq8ZnjagxptAl6jlZOaw", + "tableQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#_s2Ws5RVxAArzAZlmghwjHNGq8ZnjagxptAl6jlZOaw#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + "tableLabel": "IPL_Matches_2022", + }, + ], + } + ] + }, + }, + { + "status_code": 200, + "json": { + "resources": [ + { + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok", + "lineage": [ + { + "resourceLabel": "3fcbce17de9e55774ddedc345532c419.qvd", + "resourceQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#JOKG8u7CvizvGXwrFsyXRU0yKr2rL2WFD5djpH9bj5Q", + "tableQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#JOKG8u7CvizvGXwrFsyXRU0yKr2rL2WFD5djpH9bj5Q#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE", + "tableLabel": "test_table", + } + ], + } + ] + }, + }, + ], + }, + } + + api_vs_response.update(override_data) + + for url in api_vs_response.keys(): + if api_vs_response[url].get("response_list"): + request_mock.register_uri( + api_vs_response[url]["method"], + url, + response_list=api_vs_response[url]["response_list"], + ) + else: + request_mock.register_uri( + api_vs_response[url]["method"], + url, + json=api_vs_response[url]["json"], + status_code=api_vs_response[url]["status_code"], + ) + + +def mock_websocket_response(*args, **kwargs): + request = kwargs["request"] + if request == { + "jsonrpc": "2.0", + "id": 1, + "handle": -1, + "method": "OpenDoc", + "params": {"qDocName": "f0714ca7-7093-49e4-8b58-47bb38563647"}, + }: + return { + "qReturn": { + "qType": "Doc", + "qHandle": 1, + "qGenericId": "f0714ca7-7093-49e4-8b58-47bb38563647", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 2, + "handle": 1, + "method": "GetAppLayout", + "params": {}, + }: + return { + "qLayout": { + "qTitle": "IPL_Matches_2022", + "qFileName": "f0714ca7-7093-49e4-8b58-47bb38563647", + "qLastReloadTime": "2024-01-15T11:06:53.070Z", + "qHasScript": True, + "qStateNames": [], + "qMeta": {}, + "qHasData": True, + "qThumbnail": {}, + "qUnsupportedFeatures": [], + "qUsage": "ANALYTICS", + "encrypted": True, + "id": "f0714ca7-7093-49e4-8b58-47bb38563647", + "published": False, + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "createdDate": "2024-01-09T18:05:36.545Z", + "modifiedDate": "2024-01-15T11:07:00.333Z", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "hassectionaccess": False, + "_resourcetype": "app", + "privileges": [ + "read", + "update", + "delete", + "reload", + "export", + "duplicate", + "change_owner", + "change_space", + "export_reduced", + "source", + "exportappdata", + ], + } + } + elif request == { + "jsonrpc": "2.0", + "id": 3, + "handle": 1, + "method": "GetObjects", + "params": {"qOptions": {"qTypes": ["sheet"]}}, + }: + return { + "qList": [ + { + "qInfo": { + "qId": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "qType": "sheet", + }, + "qMeta": { + "title": "New ds sheet", + "description": "", + "_resourcetype": "app.object", + "_objecttype": "sheet", + "id": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "approved": False, + "published": False, + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "createdDate": "2024-01-15T11:01:49.704Z", + "modifiedDate": "2024-01-29T12:23:46.868Z", + "publishTime": True, + "privileges": [ + "read", + "update", + "delete", + "publish", + "change_owner", + ], + }, + "qData": {}, + }, + ] + } + elif request == { + "jsonrpc": "2.0", + "id": 4, + "handle": 1, + "method": "GetObject", + "params": {"qId": "f4f57386-263a-4ec9-b40c-abcd2467f423"}, + }: + return { + "qReturn": { + "qType": "GenericObject", + "qHandle": 2, + "qGenericType": "sheet", + "qGenericId": "f4f57386-263a-4ec9-b40c-abcd2467f423", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 5, + "handle": 2, + "method": "GetLayout", + "params": {}, + }: + return { + "qLayout": { + "qInfo": { + "qId": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "qType": "sheet", + }, + "qMeta": { + "title": "New ds sheet", + "description": "", + "_resourcetype": "app.object", + "_objecttype": "sheet", + "id": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "approved": False, + "published": False, + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "createdDate": "2024-01-15T11:01:49.704Z", + "modifiedDate": "2024-01-29T12:23:46.868Z", + "publishTime": True, + "privileges": [ + "read", + "update", + "delete", + "publish", + "change_owner", + ], + }, + "qSelectionInfo": {}, + "rank": 0, + "thumbnail": {"qStaticContentUrl": {}}, + "columns": 24, + "rows": 12, + "qChildList": { + "qItems": [ + { + "qInfo": {"qId": "QYUUb", "qType": "barchart"}, + "qMeta": {"privileges": ["read", "update", "delete"]}, + "qData": {"title": ""}, + } + ] + }, + "customRowBase": 12, + "gridResolution": "small", + "layoutOptions": {"mobileLayout": "LIST", "extendable": False}, + "gridMode": "simpleEdit", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 6, + "handle": 2, + "method": "GetChild", + "params": {"qId": "QYUUb"}, + }: + return { + "qReturn": { + "qType": "GenericObject", + "qHandle": 3, + "qGenericType": "scatterplot", + "qGenericId": "QYUUb", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 7, + "handle": 3, + "method": "GetLayout", + "params": {}, + }: + return { + "qLayout": { + "qInfo": {"qId": "QYUUb", "qType": "scatterplot"}, + "qMeta": {"privileges": ["read", "update", "delete"]}, + "qSelectionInfo": {}, + "qHyperCube": { + "qSize": {"qcx": 3, "qcy": 5}, + "qDimensionInfo": [ + { + "qFallbackTitle": "City", + "qApprMaxGlyphCount": 11, + "qCardinal": 5, + "qSortIndicator": "A", + "qGroupFallbackTitles": ["City"], + "qGroupPos": 0, + "qStateCounts": { + "qLocked": 0, + "qSelected": 0, + "qOption": 5, + "qDeselected": 0, + "qAlternative": 0, + "qExcluded": 0, + "qSelectedExcluded": 0, + "qLockedExcluded": 0, + }, + "qTags": [ + "$ascii", + "$text", + "$geoname", + "$relates_IPL_Matches_2022.City_GeoInfo", + ], + "qDimensionType": "D", + "qGrouping": "N", + "qNumFormat": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + "qIsAutoFormat": True, + "qGroupFieldDefs": ["City"], + "qMin": "NaN", + "qMax": "NaN", + "qAttrExprInfo": [], + "qAttrDimInfo": [], + "qCardinalities": { + "qCardinal": 5, + "qHypercubeCardinal": 5, + "qAllValuesCardinal": -1, + }, + "autoSort": True, + "cId": "FdErA", + "othersLabel": "Others", + } + ], + "qMeasureInfo": [ + { + "qFallbackTitle": "Sum(Date)", + "qApprMaxGlyphCount": 7, + "qCardinal": 0, + "qSortIndicator": "D", + "qNumFormat": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + "qMin": 89411, + "qMax": 2144260, + "qIsAutoFormat": True, + "qAttrExprInfo": [], + "qAttrDimInfo": [], + "qTrendLines": [], + "autoSort": True, + "cId": "PtTpyG", + "numFormatFromTemplate": True, + }, + ], + }, + "script": "", + "showTitles": True, + "title": "Test_chart", + "subtitle": "", + "footnote": "", + "disableNavMenu": False, + "showDetails": True, + "showDetailsExpression": False, + "visualization": "scatterplot", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 8, + "handle": 1, + "method": "GetObject", + "params": ["LoadModel"], + }: + return { + "qReturn": { + "qType": "GenericObject", + "qHandle": 4, + "qGenericType": "LoadModel", + "qGenericId": "LoadModel", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 9, + "handle": 4, + "method": "GetLayout", + "params": {}, + }: + return { + "qLayout": { + "qInfo": {"qId": "LoadModel", "qType": "LoadModel"}, + "tables": [ + { + "dataconnectorName": "Google_BigQuery_harshal-playground-306419", + "dataconnectorPrefix": "test_space:", + "boxType": "blackbox", + "databaseName": "", + "ownerName": "", + "tableName": "test_table", + "tableAlias": "test_table", + "loadProperties": { + "filterInfo": {"filterClause": "", "filterType": 1} + }, + "key": "Google_BigQuery_harshal-playground-306419:::test_table", + "fields": [ + { + "alias": "name", + "name": "name", + "selected": True, + "checked": True, + "id": "dsd.test_table.name", + } + ], + "connectionInfo": { + "name": "Google_BigQuery_harshal-playground-306419", + "displayName": "Google_BigQuery_harshal-playground-306419", + "id": "bb5be407-d3d3-4f19-858c-e71d593f09ae", + "type": { + "provider": "QvOdbcConnectorPackage.exe", + "type": "custom", + "name": "QvOdbcConnectorPackage", + "displayName": "Qlik® ODBC Connector Package", + "isStandardConnector": False, + "isIframeCompatible": True, + "needsConnect": True, + "connectDialog": "/customdata/64/QvOdbcConnectorPackage/web/standalone/connect-dialog.html?locale=en-US", + "selectDialog": "/customdata/64/QvOdbcConnectorPackage/web/standalone/select-dialog.html?locale=en-US", + "selectAddData": "/customdata/64/QvOdbcConnectorPackage/web/standalone/select-adddata.html?locale=en-US", + "credentialsDialog": "/customdata/64/QvOdbcConnectorPackage/web/standalone/credentials-dialog.html?locale=en-US", + "update": "/customdata/64/QvOdbcConnectorPackage/web/standalone/loadModelUpdate.js", + "architecture": {"text": "Common.undefinedbit"}, + "connectorMain": "QvOdbcConnectorPackage.webroot/connector-main-iframe", + }, + "typeName": "QvOdbcConnectorPackage.exe", + "privileges": [ + "change_owner", + "change_space", + "delete", + "list", + "read", + "update", + ], + "sourceConnectorID": "gbq", + "dataconnectorPrefix": "test_space:", + "isInAppSpace": True, + "space": "659d0e41d1b0ecce6eebc9b1", + "connectionString": 'CUSTOM CONNECT TO "provider=QvOdbcConnectorPackage.exe;driver=gbq;OAuthMechanism=0;SupportOldClient=true;Catalog_Old=harshal-playground-306419;separateCredentials=false;Catalog=harshal-playground-306419;Min_TLS=1.2;SQLDialect=1;RowsFetchedPerBlock=16384;DefaultStringColumnLength=65535;AllowLargeResults=false;EnableHTAPI=false;HTAPI_MinResultsSize=1000;HTAPI_MinActivationRatio=3;allowNonSelectQueries=false;QueryTimeout=30;Timeout=300;useBulkReader=true;bulkFetchSize=50;rowBatchSize=1;bulkFetchColumnMode=true;maxStringLength=4096;logSQLStatements=false;"', + "hasEditableSeparatedCredentials": False, + "canDelete": True, + "connectorImagePath": "https://iq37k6byr9lgam8.us.qlikcloud.com/customdata/64/QvOdbcConnectorPackage/web/gbq-square.png", + "connectorDisplayName": "Google BigQuery", + "dbInfo": {}, + }, + "id": "dsd.test_table", + "tableGroupId": "", + "connectorProperties": { + "tableQualifiers": [ + "harshal-playground-306419", + "test_dataset", + ], + }, + "selectStatement": "SELECT name\nFROM `harshal-playground-306419`.`test_dataset`.`test_table`;", + "caching": {"enabled": True, "type": "qvd"}, + }, + { + "dataconnectorName": "DataFiles", + "dataconnectorPrefix": "test_space:", + "boxType": "load-file", + "databaseName": "IPL_Matches_2022.csv", + "ownerName": "TYPE9_CSV", + "tableName": "IPL_Matches_2022", + "tableAlias": "IPL_Matches_2022", + "key": "DataFiles:IPL_Matches_2022.csv:TYPE9_CSV:IPL_Matches_2022", + "fields": [ + { + "id": "dsd.IPL_Matches_2022.City", + "name": "City", + "alias": "City", + "selected": True, + }, + { + "id": "dsd.IPL_Matches_2022.Date", + "name": "Date", + "alias": "Date", + "selected": True, + }, + ], + "connectionInfo": { + "type": {"isStorageProvider": False}, + "id": "87d7bc7e-77d8-40dc-a251-3a35ec107b4e", + "name": "DataFiles", + "typeName": "qix-datafiles.exe", + "sourceConnectorID": "qix-datafiles.exe", + "connectionString": 'CUSTOM CONNECT TO "provider=qix-datafiles.exe;path=test_space:datafiles;"', + "space": "659d0e41d1b0ecce6eebc9b1", + "dataconnectorPrefix": "test_space:", + "caching": {"enabled": True, "type": "qvd"}, + }, + "id": "dsd.IPL_Matches_2022", + "loadSelectStatement": "LOAD [ID] AS [ID],\n\t[City] AS [City],\n\t[Date] AS [Date],\n\t[Season] AS [Season],\n\t[MatchNumber] AS [MatchNumber],\n\t[Team1] AS [Team1],\n\t[Team2] AS [Team2],\n\t[Venue] AS [Venue],\n\t[TossWinner] AS [TossWinner],\n\t[TossDecision] AS [TossDecision],\n\t[SuperOver] AS [SuperOver],\n\t[WinningTeam] AS [WinningTeam],\n\t[WonBy] AS [WonBy],\n\t[Margin] AS [Margin],\n\t[method] AS [method],\n\t[Player_of_Match] AS [Player_of_Match],\n\t[Team1Players] AS [Team1Players],\n\t[Team2Players] AS [Team2Players],\n\t[Umpire1] AS [Umpire1],\n\t[Umpire2] AS [Umpire2]\nFROM [lib://DataFiles/IPL_Matches_2022.csv]\n(txt, codepage is 28591, embedded labels, delimiter is ',', msq);\n", + "formatSpec": "(txt, codepage is 28591, embedded labels, delimiter is ',', msq)", + "caching": {"enabled": True, "type": "qvd"}, + }, + ], + "schemaVersion": 2.1, + } + } + else: + return {} + + +@pytest.fixture(scope="module") +def mock_websocket_send_request(): + with patch( + "datahub.ingestion.source.qlik_sense.qlik_api.WebsocketConnection._send_request" + ) as mock_websocket_send_request, patch( + "datahub.ingestion.source.qlik_sense.websocket_connection.create_connection" + ): + mock_websocket_send_request.side_effect = mock_websocket_response + yield mock_websocket_send_request + + +def default_config(): + return { + "tenant_hostname": "iq37k6byr9lgam8.us.qlikcloud.com", + "api_key": "qlik-api-key", + "space_pattern": {"allow": ["test_space", "personal_space"]}, + } + + +@pytest.mark.integration +def test_qlik_sense_ingest( + pytestconfig, tmp_path, requests_mock, mock_websocket_send_request +): + + test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_sense" + + register_mock_api(request_mock=requests_mock) + + output_path: str = f"{tmp_path}/qlik_sense_ingest_mces.json" + + pipeline = Pipeline.create( + { + "run_id": "qlik-sense-test", + "source": { + "type": "qlik-sense", + "config": { + **default_config(), + }, + }, + "sink": { + "type": "file", + "config": { + "filename": output_path, + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_qlik_sense_ingest.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_path, + golden_path=f"{test_resources_dir}/{golden_file}", + ) + + +@pytest.mark.integration +def test_platform_instance_ingest( + pytestconfig, tmp_path, requests_mock, mock_websocket_send_request +): + + test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_sense" + + register_mock_api(request_mock=requests_mock) + + output_path: str = f"{tmp_path}/qlik_platform_instace_ingest_mces.json" + + pipeline = Pipeline.create( + { + "run_id": "qlik-sense-test", + "source": { + "type": "qlik-sense", + "config": { + **default_config(), + "platform_instance": "qlik_sense_platform", + "data_connection_to_platform_instance": { + "Google_BigQuery_harshal-playground-306419": { + "platform_instance": "google-cloud", + "env": "DEV", + } + }, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": output_path, + }, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_platform_instance_ingest.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_path, + golden_path=f"{test_resources_dir}/{golden_file}", + ) diff --git a/metadata-service/war/src/main/resources/boot/data_platforms.json b/metadata-service/war/src/main/resources/boot/data_platforms.json index 5bff689e0a555..6ef8bbc654585 100644 --- a/metadata-service/war/src/main/resources/boot/data_platforms.json +++ b/metadata-service/war/src/main/resources/boot/data_platforms.json @@ -605,6 +605,16 @@ "logoUrl": "/assets/platforms/csv-logo.png" } }, + { + "urn": "urn:li:dataPlatform:qlik-sense", + "aspect": { + "datasetNameDelimiter": ".", + "name": "qlik-sense", + "displayName": "Qlik Sense", + "type": "OTHERS", + "logoUrl": "/assets/platforms/qliklogo.png" + } + }, { "urn": "urn:li:dataPlatform:file", "aspect": {