diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts index 3784201c71c0e..27674f6dc38e1 100644 --- a/datahub-web-react/src/app/ingest/source/builder/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts @@ -31,6 +31,7 @@ import mlflowLogo from '../../../../images/mlflowlogo.png'; import dynamodbLogo from '../../../../images/dynamodblogo.png'; import fivetranLogo from '../../../../images/fivetranlogo.png'; import csvLogo from '../../../../images/csv-logo.png'; +import qlikLogo from '../../../../images/qliklogo.png'; export const ATHENA = 'athena'; export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`; @@ -113,6 +114,8 @@ export const FIVETRAN = 'fivetran'; export const FIVETRAN_URN = `urn:li:dataPlatform:${FIVETRAN}`; export const CSV = 'csv-enricher'; export const CSV_URN = `urn:li:dataPlatform:${CSV}`; +export const QLIK_SENSE = 'qlik-sense'; +export const QLIK_SENSE_URN = `urn:li:dataPlatform:${QLIK_SENSE}`; export const PLATFORM_URN_TO_LOGO = { [ATHENA_URN]: athenaLogo, @@ -149,6 +152,7 @@ export const PLATFORM_URN_TO_LOGO = { [VERTICA_URN]: verticaLogo, [FIVETRAN_URN]: fivetranLogo, [CSV_URN]: csvLogo, + [QLIK_SENSE_URN]: qlikLogo, }; export const SOURCE_TO_PLATFORM_URN = { diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index e33de13c100b7..f4bfd45eb4f83 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -236,5 +236,12 @@ "displayName": "Other", "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/", "recipe": "source:\n type: \n config:\n # Source-type specifics config\n " + }, + { + "urn": "urn:li:dataPlatform:qlik-sense", + "name": "qlik-sense", + "displayName": "Qlik Sense", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/qlik-sense/", + "recipe": "source:\n type: qlik-sense\n config:\n # Coordinates\n tenant_hostname: https://xyz12xz.us.qlikcloud.com\n # Coordinates\n api_key: QLIK_API_KEY\n\n # Optional - filter for certain space names instead of ingesting everything.\n # space_pattern:\n\n # allow:\n # - space_name\n ingest_owner: true" } ] diff --git a/datahub-web-react/src/images/qliklogo.png b/datahub-web-react/src/images/qliklogo.png new file mode 100644 index 0000000000000..fe69faed8b9a9 Binary files /dev/null and b/datahub-web-react/src/images/qliklogo.png differ diff --git a/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_pre.md b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_pre.md new file mode 100644 index 0000000000000..c344a32d07676 --- /dev/null +++ b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_pre.md @@ -0,0 +1,23 @@ +## Integration Details + +This source extracts the following: + +- Accessible spaces and apps within that spaces as Container. +- Qlik Datasets as Datahub Datasets with schema metadata. +- Sheets as Datahub dashboard and charts present inside sheets. + +## Configuration Notes + +1. Refer [doc](https://qlik.dev/authenticate/api-key/generate-your-first-api-key/) to generate an API key from the hub. +2. Get tenant hostname from About tab after login to qlik sense account. + +## Concept mapping + +| Qlik Sense | Datahub | Notes | +|------------------------|---------------------------------------------------------------|----------------------------------| +| `Space` | [Container](../../metamodel/entities/container.md) | SubType `"Qlik Space"` | +| `App` | [Container](../../metamodel/entities/container.md) | SubType `"Qlik App"` | +| `Sheet` | [Dashboard](../../metamodel/entities/dashboard.md) | | +| `Chart` | [Chart](../../metamodel/entities/chart.md) | | +| `Dataset` | [Dataset](../../metamodel/entities/dataset.md) | SubType `"Qlik Dataset"` | +| `User` | [User (a.k.a CorpUser)](../../metamodel/entities/corpuser.md) | Optionally Extracted | \ No newline at end of file diff --git a/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml new file mode 100644 index 0000000000000..315e3d1de3f7f --- /dev/null +++ b/metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml @@ -0,0 +1,25 @@ +source: + type: qlik-sense + config: + # Coordinates + tenant_hostname: "xyz12xz.us.qlikcloud.com" + # Credentials + api_key: "QLIK_API_KEY" + + # Optional - filter for certain space names instead of ingesting everything. + # Mention 'personal_space' if entities of personal space need to ingest + # space_pattern: + # allow: + # - space_name + + ingest_owner: true + + # Optional -- This mapping is optional and only required to configure platform-instance for Qlik app dataset upstream source tables + # A mapping of the Qlik app dataset upstream tables from data connection to platform instance. Use 'data_connection_name' as key. + # data_connection_to_platform_instance: + # data_connection_name: + # platform_instance: cloud_instance + # env: DEV + +sink: + # sink configs diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 6f0e4d8fb6f37..3c6aafec26185 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -393,6 +393,7 @@ # databricks is alias for unity-catalog and needs to be kept in sync "databricks": databricks | sql_common | sqllineage_lib, "fivetran": snowflake_common | bigquery_common, + "qlik-sense": sqlglot_lib | {"requests", "websocket-client"}, } # This is mainly used to exclude plugins from the Docker image. @@ -521,6 +522,7 @@ "mode", "fivetran", "kafka-connect", + "qlik-sense", ] if plugin for dependency in plugins[plugin] @@ -625,6 +627,7 @@ "gcs = datahub.ingestion.source.gcs.gcs_source:GCSSource", "sql-queries = datahub.ingestion.source.sql_queries:SqlQueriesSource", "fivetran = datahub.ingestion.source.fivetran.fivetran:FivetranSource", + "qlik-sense = datahub.ingestion.source.qlik_sense.qlik_sense:QlikSenseSource", ], "datahub.ingestion.transformer.plugins": [ "simple_remove_dataset_ownership = datahub.ingestion.transformer.remove_dataset_ownership:SimpleRemoveDatasetOwnership", diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py index 3296a8fb29354..1def0dd02097b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py @@ -15,6 +15,7 @@ class DatasetSubTypes(str, Enum): SALESFORCE_CUSTOM_OBJECT = "Custom Object" SALESFORCE_STANDARD_OBJECT = "Object" POWERBI_DATASET_TABLE = "PowerBI Dataset Table" + QLIK_DATASET = "Qlik Dataset" BIGQUERY_TABLE_SNAPSHOT = "Bigquery Table Snapshot" # TODO: Create separate entity... @@ -40,6 +41,8 @@ class BIContainerSubTypes(str, Enum): TABLEAU_WORKBOOK = "Workbook" POWERBI_WORKSPACE = "Workspace" POWERBI_DATASET = "PowerBI Dataset" + QLIK_SPACE = "Qlik Space" + QLIK_APP = "Qlik App" class BIAssetSubTypes(str, Enum): diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py new file mode 100644 index 0000000000000..eb0b9d02d865e --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py @@ -0,0 +1,130 @@ +import logging +from dataclasses import dataclass +from typing import Dict, Optional + +import pydantic + +from datahub.configuration.common import AllowDenyPattern +from datahub.configuration.source_common import ( + EnvConfigMixin, + PlatformInstanceConfigMixin, +) +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalSourceReport, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionConfigBase, +) + +logger = logging.getLogger(__name__) + +QLIK_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + + +class Constant: + """ + keys used in qlik plugin + """ + + # Rest API response key constants + DATA = "data" + ID = "id" + NAME = "name" + TYPE = "type" + ITEMID = "itemId" + NEXT = "next" + LINKS = "links" + HREF = "href" + DATASETTYPE = "datasetType" + CREATEDAT = "createdAt" + UPDATEDAT = "updatedAt" + SECUREQRI = "secureQri" + QRI = "qri" + SPACEID = "spaceId" + SPACE = "space" + CREATEDTIME = "createdTime" + LASTMODIFIEDTIME = "lastModifiedTime" + OPERATIONAL = "operational" + SIZE = "size" + ROWCOUNT = "rowCount" + DATATYPE = "dataType" + PRIMARYKEY = "primaryKey" + NULLABLE = "nullable" + SCHEMA = "schema" + DATAFIELDS = "dataFields" + RESOURCETYPE = "resourceType" + USAGE = "usage" + CREATEDDATE = "createdDate" + MODIFIEDDATE = "modifiedDate" + RESOURCEID = "resourceId" + DATASETSCHEMA = "datasetSchema" + GRAPH = "graph" + NODES = "nodes" + RESOURCES = "resources" + LINEAGE = "lineage" + TABLELABEL = "tableLabel" + TABLEQRI = "tableQRI" + OWNERID = "ownerId" + # Websocket response key constants + QID = "qId" + RESULT = "result" + QRETURN = "qReturn" + QTYPE = "qType" + QHANDLE = "qHandle" + QLAYOUT = "qLayout" + QMETA = "qMeta" + QCHILDLIST = "qChildList" + QITEMS = "qItems" + QINFO = "qInfo" + QLIST = "qList" + CONNECTORPROPERTIES = "connectorProperties" + TABLEQUALIFIERS = "tableQualifiers" + CONNECTIONINFO = "connectionInfo" + SOURCECONNECTORID = "sourceConnectorID" + DATABASENAME = "databaseName" + SCHEMANAME = "schemaName" + TABLES = "tables" + DATACONNECTORID = "dataconnectorid" + DATACONNECTORNAME = "dataconnectorName" + DATACONNECTORPLATFORM = "dataconnectorPlatform" + # Item type + APP = "app" + DATASET = "dataset" + # Personal entity constants + PERSONAL_SPACE_ID = "personal-space-id" + PERSONAL_SPACE_NAME = "personal_space" + + +@dataclass +class QlikSourceReport(StaleEntityRemovalSourceReport): + number_of_spaces: int = 0 + + def report_number_of_spaces(self, number_of_spaces: int) -> None: + self.number_of_spaces = number_of_spaces + + +class PlatformDetail(PlatformInstanceConfigMixin, EnvConfigMixin): + pass + + +class QlikSourceConfig( + StatefulIngestionConfigBase, PlatformInstanceConfigMixin, EnvConfigMixin +): + tenant_hostname: str = pydantic.Field(description="Qlik Tenant hostname") + api_key: str = pydantic.Field(description="Qlik API Key") + # Qlik space identifier + space_pattern: AllowDenyPattern = pydantic.Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns to filter Qlik spaces in ingestion." + "Mention 'personal_space' if entities of personal space need to ingest", + ) + ingest_owner: Optional[bool] = pydantic.Field( + default=True, + description="Ingest Owner from source. This will override Owner info entered from UI", + ) + # Qlik app dataset upstream tables from data connection to platform instance mapping + data_connection_to_platform_instance: Dict[str, PlatformDetail] = pydantic.Field( + default={}, + description="A mapping of the Qlik app dataset upstream tables from data connection to platform instance." + "Use 'data_connection_name' as key.", + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py new file mode 100644 index 0000000000000..c30b456253e06 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py @@ -0,0 +1,235 @@ +from datetime import datetime +from enum import Enum +from typing import Dict, List, Optional, Type, Union + +from pydantic import BaseModel, Field, root_validator + +from datahub.emitter.mcp_builder import ContainerKey +from datahub.ingestion.source.qlik_sense.config import QLIK_DATETIME_FORMAT, Constant +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + BooleanType, + BytesType, + DateType, + NumberType, + StringType, + TimeType, +) + +FIELD_TYPE_MAPPING: Dict[ + str, + Type[ + Union[ + BooleanType, + BytesType, + DateType, + NumberType, + StringType, + TimeType, + ] + ], +] = { + "DATE": DateType, + "TIME": TimeType, + "DATETIME": DateType, + "TIMESTAMP": DateType, + "STRING": StringType, + "DOUBLE": NumberType, + "DECIMAL": NumberType, + "INTEGER": NumberType, + "BOOLEAN": BooleanType, + "BINARY": BytesType, +} + +KNOWN_DATA_PLATFORM_MAPPING = { + "gbq": "bigquery", + "snowflake": "snowflake", +} + + +class SpaceKey(ContainerKey): + space: str + + +class AppKey(ContainerKey): + app: str + + +class SpaceType(Enum): + PERSONAL = "personal" + SHARED = "shared" + MANAGED = "managed" + DATA = "data" + + +# Qlik table box type +class BoxType(Enum): + LOADFILE = "load-file" # Table extracted from dataset + BLACKBOX = "blackbox" # Table extracted from data connection + + +PERSONAL_SPACE_DICT = { + "id": Constant.PERSONAL_SPACE_ID, + "name": Constant.PERSONAL_SPACE_NAME, + "description": "", + "type": SpaceType.PERSONAL, + "createdAt": datetime.now().strftime(QLIK_DATETIME_FORMAT), + "updatedAt": datetime.now().strftime(QLIK_DATETIME_FORMAT), +} + + +class Space(BaseModel): + id: str + name: str + description: str + type: SpaceType + createdAt: datetime + updatedAt: datetime + ownerId: Optional[str] = None + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.CREATEDAT] = datetime.strptime( + values[Constant.CREATEDAT], QLIK_DATETIME_FORMAT + ) + values[Constant.UPDATEDAT] = datetime.strptime( + values[Constant.UPDATEDAT], QLIK_DATETIME_FORMAT + ) + return values + + +class Item(BaseModel): + id: str + description: str = "" + ownerId: str + spaceId: str + createdAt: datetime + updatedAt: datetime + + +class SchemaField(BaseModel): + name: str + dataType: Optional[str] = None + primaryKey: Optional[bool] = None + nullable: Optional[bool] = None + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.DATATYPE] = values.get(Constant.DATATYPE, {}).get(Constant.TYPE) + return values + + +class QlikDataset(Item): + name: str + secureQri: str + type: str + size: int + rowCount: int + itemId: str + datasetSchema: List[SchemaField] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + # Update str time to datetime + values[Constant.CREATEDAT] = datetime.strptime( + values[Constant.CREATEDTIME], QLIK_DATETIME_FORMAT + ) + values[Constant.UPDATEDAT] = datetime.strptime( + values[Constant.LASTMODIFIEDTIME], QLIK_DATETIME_FORMAT + ) + if not values.get(Constant.SPACEID): + # spaceId none indicates dataset present in personal space + values[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID + values[Constant.QRI] = values[Constant.SECUREQRI] + values[Constant.SIZE] = values[Constant.OPERATIONAL].get(Constant.SIZE, 0) + values[Constant.ROWCOUNT] = values[Constant.OPERATIONAL][Constant.ROWCOUNT] + + values[Constant.DATASETSCHEMA] = values[Constant.SCHEMA][Constant.DATAFIELDS] + return values + + +class AxisProperty(BaseModel): + Title: str = Field(alias="qFallbackTitle") + Min: str = Field(alias="qMin") + Max: str = Field(alias="qMax") + + +class Chart(BaseModel): + qId: str + visualization: str + title: str + subtitle: str + qDimension: List[AxisProperty] + qMeasure: List[AxisProperty] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.QID] = values[Constant.QINFO][Constant.QID] + values["qDimension"] = values["qHyperCube"]["qDimensionInfo"] + values["qMeasure"] = values["qHyperCube"]["qMeasureInfo"] + return values + + +class Sheet(BaseModel): + id: str + title: str + description: str + ownerId: str + createdAt: datetime + updatedAt: datetime + charts: List[Chart] = [] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.CREATEDAT] = datetime.strptime( + values[Constant.CREATEDDATE], QLIK_DATETIME_FORMAT + ) + values[Constant.UPDATEDAT] = datetime.strptime( + values[Constant.MODIFIEDDATE], QLIK_DATETIME_FORMAT + ) + return values + + +class QlikTable(BaseModel): + tableName: str + type: BoxType = Field(alias="boxType") + tableAlias: str + dataconnectorid: str + dataconnectorName: str + dataconnectorPlatform: str + spaceId: str + datasetSchema: List[SchemaField] = Field(alias="fields") + tableQri: Optional[str] = None + selectStatement: Optional[str] = None + databaseName: Optional[str] = None + schemaName: Optional[str] = None + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.DATACONNECTORID] = values[Constant.CONNECTIONINFO][Constant.ID] + values[Constant.DATACONNECTORPLATFORM] = values[Constant.CONNECTIONINFO][ + Constant.SOURCECONNECTORID + ] + values[Constant.SPACEID] = values[Constant.CONNECTIONINFO][Constant.SPACE] + return values + + +class App(Item): + qTitle: str + qri: str + qUsage: str + sheets: List[Sheet] = [] + tables: List[QlikTable] = [] + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + values[Constant.CREATEDAT] = datetime.strptime( + values[Constant.CREATEDDATE], QLIK_DATETIME_FORMAT + ) + values[Constant.UPDATEDAT] = datetime.strptime( + values[Constant.MODIFIEDDATE], QLIK_DATETIME_FORMAT + ) + if not values.get(Constant.SPACEID): + # spaceId none indicates app present in personal space + values[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID + values[Constant.QRI] = f"qri:app:sense://{values['id']}" + return values diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py new file mode 100644 index 0000000000000..abcd0c0ce8353 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py @@ -0,0 +1,300 @@ +import logging +import sys +from typing import Any, Dict, List, Optional +from urllib.parse import quote + +import requests + +from datahub.ingestion.source.qlik_sense.config import Constant, QlikSourceConfig +from datahub.ingestion.source.qlik_sense.data_classes import ( + PERSONAL_SPACE_DICT, + App, + Chart, + Item, + QlikDataset, + QlikTable, + Sheet, + Space, +) +from datahub.ingestion.source.qlik_sense.websocket_connection import WebsocketConnection + +# Logger instance +logger = logging.getLogger(__name__) + + +class QlikAPI: + def __init__(self, config: QlikSourceConfig) -> None: + self.spaces: Dict = {} + self.users: Dict = {} + self.config = config + self.session = requests.Session() + self.session.headers.update( + { + "Authorization": f"Bearer {self.config.api_key}", + "Content-Type": "application/json", + } + ) + self.rest_api_url = f"https://{self.config.tenant_hostname}/api/v1" + # Test connection by fetching list of api keys + logger.info("Trying to connect to {}".format(self.rest_api_url)) + self.session.get(f"{self.rest_api_url}/api-keys").raise_for_status() + + def _log_http_error(self, message: str) -> Any: + logger.warning(message) + _, e, _ = sys.exc_info() + if isinstance(e, requests.exceptions.HTTPError): + logger.warning(f"HTTP status-code = {e.response.status_code}") + logger.debug(msg=message, exc_info=e) + return e + + def get_spaces(self) -> List[Space]: + spaces: List[Space] = [] + try: + response = self.session.get(f"{self.rest_api_url}/spaces") + response.raise_for_status() + for space_dict in response.json()[Constant.DATA]: + space = Space.parse_obj(space_dict) + spaces.append(space) + self.spaces[space.id] = space.name + # Add personal space entity + spaces.append(Space.parse_obj(PERSONAL_SPACE_DICT)) + self.spaces[PERSONAL_SPACE_DICT[Constant.ID]] = PERSONAL_SPACE_DICT[ + Constant.NAME + ] + except Exception as e: + self._log_http_error(message=f"Unable to fetch spaces. Exception: {e}") + return spaces + + def _get_dataset(self, dataset_id: str, item_id: str) -> Optional[QlikDataset]: + try: + response = self.session.get(f"{self.rest_api_url}/data-sets/{dataset_id}") + response.raise_for_status() + response_dict = response.json() + response_dict[Constant.ITEMID] = item_id + return QlikDataset.parse_obj(response_dict) + except Exception as e: + self._log_http_error( + message=f"Unable to fetch dataset with id {dataset_id}. Exception: {e}" + ) + return None + + def get_user_name(self, user_id: str) -> Optional[str]: + try: + if user_id in self.users: + # To avoid fetching same user details again + return self.users[user_id] + else: + response = self.session.get(f"{self.rest_api_url}/users/{user_id}") + response.raise_for_status() + user_name = response.json()[Constant.NAME] + self.users[user_id] = user_name + return user_name + except Exception as e: + self._log_http_error( + message=f"Unable to fetch user with id {user_id}. Exception: {e}" + ) + return None + + def _get_chart( + self, + websocket_connection: WebsocketConnection, + chart_id: str, + sheet_id: str, + ) -> Optional[Chart]: + try: + websocket_connection.websocket_send_request( + method="GetChild", params={"qId": chart_id} + ) + response = websocket_connection.websocket_send_request(method="GetLayout") + return Chart.parse_obj(response[Constant.QLAYOUT]) + except Exception as e: + self._log_http_error( + message=f"Unable to fetch chart {chart_id} of sheet {sheet_id}. Exception: {e}" + ) + return None + + def _get_sheet( + self, + websocket_connection: WebsocketConnection, + sheet_id: str, + ) -> Optional[Sheet]: + try: + websocket_connection.websocket_send_request( + method="GetObject", params={"qId": sheet_id} + ) + response = websocket_connection.websocket_send_request(method="GetLayout") + sheet_dict = response[Constant.QLAYOUT] + if Constant.OWNERID not in sheet_dict[Constant.QMETA]: + # That means sheet is private sheet + return None + sheet = Sheet.parse_obj(sheet_dict[Constant.QMETA]) + for i, chart_dict in enumerate( + sheet_dict[Constant.QCHILDLIST][Constant.QITEMS] + ): + chart = self._get_chart( + websocket_connection, + chart_dict[Constant.QINFO][Constant.QID], + sheet_id, + ) + if chart: + if not chart.title: + chart.title = f"Object {i+1} of Sheet '{sheet.title}'" + sheet.charts.append(chart) + websocket_connection.handle.pop() + return sheet + except Exception as e: + self._log_http_error( + message=f"Unable to fetch sheet with id {sheet_id}. Exception: {e}" + ) + return None + + def _add_qri_of_tables(self, tables: List[QlikTable], app_id: str) -> None: + table_qri_dict: Dict[str, str] = {} + app_qri = quote(f"qri:app:sense://{app_id}", safe="") + try: + response = self.session.get( + f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/actions/expand?node={app_qri}&level=TABLE" + ) + response.raise_for_status() + for table_node_qri in response.json()[Constant.GRAPH][Constant.NODES]: + table_node_qri = quote(table_node_qri, safe="") + response = self.session.get( + f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/actions/expand?node={table_node_qri}&level=FIELD" + ) + response.raise_for_status() + field_nodes_qris = list( + response.json()[Constant.GRAPH][Constant.NODES].keys() + ) + for field_node_qri in field_nodes_qris: + response = self.session.post( + f"{self.rest_api_url}/lineage-graphs/nodes/{app_qri}/overview", + json=[field_node_qri], + ) + response.raise_for_status() + # Some fields might not have lineage overview, in that case status code is 207 + if response.status_code == 200: + for each_lineage in response.json()[Constant.RESOURCES][0][ + Constant.LINEAGE + ]: + table_name = ( + each_lineage[Constant.TABLELABEL] + .replace('"', "") + .split(".")[-1] + ) + table_qri_dict[table_name] = each_lineage[Constant.TABLEQRI] + break + for table in tables: + if table.tableName in table_qri_dict: + table.tableQri = table_qri_dict[table.tableName] + except Exception as e: + self._log_http_error( + message=f"Unable to add QRI for tables of app {app_id}. Exception: {e}" + ) + + def _get_app_used_tables( + self, websocket_connection: WebsocketConnection, app_id: str + ) -> List[QlikTable]: + tables: List[QlikTable] = [] + try: + response = websocket_connection.websocket_send_request( + method="GetObject", + params=["LoadModel"], + ) + if not response[Constant.QRETURN][Constant.QTYPE]: + return [] + response = websocket_connection.websocket_send_request(method="GetLayout") + for table_dict in response[Constant.QLAYOUT][Constant.TABLES]: + tables.append(QlikTable.parse_obj(table_dict)) + websocket_connection.handle.pop() + self._add_qri_of_tables(tables, app_id) + except Exception as e: + self._log_http_error( + message=f"Unable to fetch tables used by app {app_id}. Exception: {e}" + ) + return tables + + def _get_app_sheets( + self, websocket_connection: WebsocketConnection, app_id: str + ) -> List[Sheet]: + sheets: List[Sheet] = [] + try: + response = websocket_connection.websocket_send_request( + method="GetObjects", + params={ + "qOptions": { + "qTypes": ["sheet"], + } + }, + ) + for sheet_dict in response[Constant.QLIST]: + sheet = self._get_sheet( + websocket_connection=websocket_connection, + sheet_id=sheet_dict[Constant.QINFO][Constant.QID], + ) + if sheet: + sheets.append(sheet) + websocket_connection.handle.pop() + except Exception as e: + self._log_http_error( + message=f"Unable to fetch sheets for app {app_id}. Exception: {e}" + ) + return sheets + + def _get_app(self, app_id: str) -> Optional[App]: + try: + websocket_connection = WebsocketConnection( + self.config.tenant_hostname, self.config.api_key, app_id + ) + websocket_connection.websocket_send_request( + method="OpenDoc", + params={"qDocName": app_id}, + ) + response = websocket_connection.websocket_send_request( + method="GetAppLayout" + ) + app = App.parse_obj(response[Constant.QLAYOUT]) + app.sheets = self._get_app_sheets(websocket_connection, app_id) + app.tables = self._get_app_used_tables(websocket_connection, app_id) + websocket_connection.close_websocket() + return app + except Exception as e: + self._log_http_error( + message=f"Unable to fetch app with id {app_id}. Exception: {e}" + ) + return None + + def get_items(self) -> List[Item]: + items: List[Item] = [] + try: + url = f"{self.rest_api_url}/items" + while True: + response = self.session.get(url) + response.raise_for_status() + response_dict = response.json() + for item in response_dict[Constant.DATA]: + # spaceId none indicates item present in personal space + if not item.get(Constant.SPACEID): + item[Constant.SPACEID] = Constant.PERSONAL_SPACE_ID + if self.config.space_pattern.allowed( + self.spaces[item[Constant.SPACEID]] + ): + resource_type = item[Constant.RESOURCETYPE] + if resource_type == Constant.APP: + app = self._get_app(app_id=item[Constant.RESOURCEID]) + if app: + items.append(app) + elif resource_type == Constant.DATASET: + dataset = self._get_dataset( + dataset_id=item[Constant.RESOURCEID], + item_id=item[Constant.ID], + ) + if dataset: + items.append(dataset) + if Constant.NEXT in response_dict[Constant.LINKS]: + url = response_dict[Constant.LINKS][Constant.NEXT][Constant.HREF] + else: + break + + except Exception as e: + self._log_http_error(message=f"Unable to fetch items. Exception: {e}") + return items diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py new file mode 100644 index 0000000000000..a5b9adae0376c --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py @@ -0,0 +1,602 @@ +import logging +from datetime import datetime +from typing import Dict, Iterable, List, Optional + +import datahub.emitter.mce_builder as builder +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp_builder import add_entity_to_container, gen_containers +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.decorators import ( + SourceCapability, + SupportStatus, + capability, + config_class, + platform_name, + support_status, +) +from datahub.ingestion.api.source import ( + CapabilityReport, + MetadataWorkUnitProcessor, + SourceReport, + TestableSource, + TestConnectionReport, +) +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.common.subtypes import ( + BIContainerSubTypes, + DatasetSubTypes, +) +from datahub.ingestion.source.qlik_sense.config import ( + Constant, + PlatformDetail, + QlikSourceConfig, + QlikSourceReport, +) +from datahub.ingestion.source.qlik_sense.data_classes import ( + FIELD_TYPE_MAPPING, + KNOWN_DATA_PLATFORM_MAPPING, + App, + AppKey, + BoxType, + Chart, + QlikDataset, + QlikTable, + SchemaField as QlikDatasetSchemaField, + Sheet, + Space, + SpaceKey, +) +from datahub.ingestion.source.qlik_sense.qlik_api import QlikAPI +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalHandler, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionSourceBase, +) +from datahub.metadata.com.linkedin.pegasus2avro.common import ( + Status, + SubTypes, + TimeStamp, +) +from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( + DatasetLineageType, + DatasetProperties, + FineGrainedLineage, + FineGrainedLineageDownstreamType, + FineGrainedLineageUpstreamType, + Upstream, + UpstreamLineage, +) +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + MySqlDDL, + NullType, + SchemaField, + SchemaMetadata, +) +from datahub.metadata.schema_classes import ( + AuditStampClass, + ChangeAuditStampsClass, + ChartInfoClass, + DashboardInfoClass, + DataPlatformInstanceClass, + OwnerClass, + OwnershipClass, + OwnershipTypeClass, + SchemaFieldDataTypeClass, +) +from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result + +# Logger instance +logger = logging.getLogger(__name__) + + +@platform_name("Qlik Sense") +@config_class(QlikSourceConfig) +@support_status(SupportStatus.INCUBATING) +@capability(SourceCapability.DESCRIPTIONS, "Enabled by default") +@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability( + SourceCapability.OWNERSHIP, + "Enabled by default, configured using `ingest_owner`", +) +class QlikSenseSource(StatefulIngestionSourceBase, TestableSource): + """ + This plugin extracts the following: + - Qlik Sense Spaces and Apps as Container. + - Qlik Datasets + - Sheets as dashboard and its charts + """ + + config: QlikSourceConfig + reporter: QlikSourceReport + platform: str = "qlik-sense" + + def __init__(self, config: QlikSourceConfig, ctx: PipelineContext): + super(QlikSenseSource, self).__init__(config, ctx) + self.config = config + self.reporter = QlikSourceReport() + try: + self.qlik_api = QlikAPI(self.config) + except Exception as e: + logger.warning(e) + exit( + 1 + ) # Exit pipeline as we are not able to connect to Qlik Client Service. + + @staticmethod + def test_connection(config_dict: dict) -> TestConnectionReport: + test_report = TestConnectionReport() + try: + QlikAPI(QlikSourceConfig.parse_obj_allow_extras(config_dict)) + test_report.basic_connectivity = CapabilityReport(capable=True) + except Exception as e: + test_report.basic_connectivity = CapabilityReport( + capable=False, failure_reason=str(e) + ) + return test_report + + @classmethod + def create(cls, config_dict, ctx): + config = QlikSourceConfig.parse_obj(config_dict) + return cls(config, ctx) + + def _gen_space_key(self, space_id: str) -> SpaceKey: + return SpaceKey( + space=space_id, + platform=self.platform, + instance=self.config.platform_instance, + ) + + def _gen_app_key(self, app_id: str) -> AppKey: + return AppKey( + app=app_id, + platform=self.platform, + instance=self.config.platform_instance, + ) + + def _get_audit_stamp(self, date: datetime, username: str) -> AuditStampClass: + return AuditStampClass( + time=int(date.timestamp() * 1000), + actor=builder.make_user_urn(username), + ) + + def _get_allowed_spaces(self) -> List[Space]: + all_spaces = self.qlik_api.get_spaces() + allowed_spaces = [ + space + for space in all_spaces + if self.config.space_pattern.allowed(space.name) + ] + logger.info(f"Number of spaces = {len(all_spaces)}") + self.reporter.report_number_of_spaces(len(all_spaces)) + logger.info(f"Number of allowed spaces = {len(allowed_spaces)}") + return allowed_spaces + + def _gen_space_workunit(self, space: Space) -> Iterable[MetadataWorkUnit]: + """ + Map Qlik space to Datahub container + """ + owner_username: Optional[str] = None + if space.ownerId: + owner_username = self.qlik_api.get_user_name(space.ownerId) + yield from gen_containers( + container_key=self._gen_space_key(space.id), + name=space.name, + description=space.description, + sub_types=[BIContainerSubTypes.QLIK_SPACE], + extra_properties={Constant.TYPE: str(space.type)}, + owner_urn=builder.make_user_urn(owner_username) + if self.config.ingest_owner and owner_username + else None, + external_url=f"https://{self.config.tenant_hostname}/catalog?space_filter={space.id}", + created=int(space.createdAt.timestamp() * 1000), + last_modified=int(space.updatedAt.timestamp() * 1000), + ) + + def _gen_entity_status_aspect(self, entity_urn: str) -> MetadataWorkUnit: + return MetadataChangeProposalWrapper( + entityUrn=entity_urn, aspect=Status(removed=False) + ).as_workunit() + + def _gen_entity_owner_aspect( + self, entity_urn: str, user_name: str + ) -> MetadataWorkUnit: + aspect = OwnershipClass( + owners=[ + OwnerClass( + owner=builder.make_user_urn(user_name), + type=OwnershipTypeClass.DATAOWNER, + ) + ] + ) + return MetadataChangeProposalWrapper( + entityUrn=entity_urn, + aspect=aspect, + ).as_workunit() + + def _gen_dashboard_urn(self, dashboard_identifier: str) -> str: + return builder.make_dashboard_urn( + platform=self.platform, + platform_instance=self.config.platform_instance, + name=dashboard_identifier, + ) + + def _gen_dashboard_info_workunit( + self, sheet: Sheet, app_id: str + ) -> MetadataWorkUnit: + dashboard_urn = self._gen_dashboard_urn(sheet.id) + custom_properties: Dict[str, str] = {"chartCount": str(len(sheet.charts))} + dashboard_info_cls = DashboardInfoClass( + title=sheet.title, + description=sheet.description, + charts=[ + builder.make_chart_urn( + platform=self.platform, + platform_instance=self.config.platform_instance, + name=chart.qId, + ) + for chart in sheet.charts + ], + lastModified=ChangeAuditStampsClass( + created=self._get_audit_stamp(sheet.createdAt, sheet.ownerId), + lastModified=self._get_audit_stamp(sheet.updatedAt, sheet.ownerId), + ), + customProperties=custom_properties, + dashboardUrl=f"https://{self.config.tenant_hostname}/sense/app/{app_id}/sheet/{sheet.id}/state/analysis", + ) + return MetadataChangeProposalWrapper( + entityUrn=dashboard_urn, aspect=dashboard_info_cls + ).as_workunit() + + def _gen_charts_workunit( + self, charts: List[Chart], input_tables: List[QlikTable], app_id: str + ) -> Iterable[MetadataWorkUnit]: + """ + Map Qlik Chart to Datahub Chart + """ + input_tables_urns: List[str] = [] + for table in input_tables: + table_identifier = self._get_app_table_identifier(table) + if table_identifier: + input_tables_urns.append(self._gen_qlik_dataset_urn(table_identifier)) + + for chart in charts: + chart_urn = builder.make_chart_urn( + platform=self.platform, + platform_instance=self.config.platform_instance, + name=chart.qId, + ) + + yield self._gen_entity_status_aspect(chart_urn) + + custom_properties = { + "Dimension": str(chart.qDimension), + "Measure": str(chart.qMeasure), + } + + yield MetadataChangeProposalWrapper( + entityUrn=chart_urn, + aspect=ChartInfoClass( + title=chart.title, + description=chart.visualization, + lastModified=ChangeAuditStampsClass(), + customProperties=custom_properties, + inputs=input_tables_urns, + ), + ).as_workunit() + + yield from add_entity_to_container( + container_key=self._gen_app_key(app_id), + entity_type="chart", + entity_urn=chart_urn, + ) + + def _gen_sheets_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: + """ + Map Qlik Sheet to Datahub dashboard + """ + for sheet in app.sheets: + dashboard_urn = self._gen_dashboard_urn(sheet.id) + + yield self._gen_entity_status_aspect(dashboard_urn) + + yield self._gen_dashboard_info_workunit(sheet, app.id) + + yield from add_entity_to_container( + container_key=self._gen_app_key(app.id), + entity_type="dashboard", + entity_urn=dashboard_urn, + ) + + dpi_aspect = self._gen_dataplatform_instance_aspect(dashboard_urn) + if dpi_aspect: + yield dpi_aspect + + owner_username = self.qlik_api.get_user_name(sheet.ownerId) + if self.config.ingest_owner and owner_username: + yield self._gen_entity_owner_aspect(dashboard_urn, owner_username) + + yield from self._gen_charts_workunit(sheet.charts, app.tables, app.id) + + def _gen_app_table_upstream_lineage( + self, dataset_urn: str, table: QlikTable + ) -> Optional[MetadataWorkUnit]: + upstream_dataset_urn: Optional[str] = None + if table.type == BoxType.BLACKBOX: + upstream_dataset_platform_detail = ( + self.config.data_connection_to_platform_instance.get( + table.dataconnectorName, + PlatformDetail( + env=self.config.env, + platform_instance=self.config.platform_instance, + ), + ) + ) + if not table.selectStatement: + return None + upstream_dataset_urn = create_lineage_sql_parsed_result( + query=table.selectStatement.strip(), + default_db=None, + platform=KNOWN_DATA_PLATFORM_MAPPING.get( + table.dataconnectorPlatform, table.dataconnectorPlatform + ), + env=upstream_dataset_platform_detail.env, + platform_instance=upstream_dataset_platform_detail.platform_instance, + ).in_tables[0] + elif table.type == BoxType.LOADFILE: + upstream_dataset_urn = self._gen_qlik_dataset_urn( + f"{table.spaceId}.{table.databaseName}".lower() + ) + + if upstream_dataset_urn: + # Generate finegrained lineage + fine_grained_lineages = [ + FineGrainedLineage( + upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, + upstreams=[ + builder.make_schema_field_urn(upstream_dataset_urn, field.name) + ], + downstreamType=FineGrainedLineageDownstreamType.FIELD, + downstreams=[ + builder.make_schema_field_urn(dataset_urn, field.name) + ], + ) + for field in table.datasetSchema + ] + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=UpstreamLineage( + upstreams=[ + Upstream( + dataset=upstream_dataset_urn, type=DatasetLineageType.COPY + ) + ], + fineGrainedLineages=fine_grained_lineages, + ), + ).as_workunit() + else: + return None + + def _gen_app_table_properties( + self, dataset_urn: str, table: QlikTable + ) -> MetadataWorkUnit: + dataset_properties = DatasetProperties( + name=table.tableName, + qualifiedName=table.tableAlias, + ) + dataset_properties.customProperties.update( + { + Constant.TYPE: "Qlik Table", + Constant.DATACONNECTORID: table.dataconnectorid, + Constant.DATACONNECTORNAME: table.dataconnectorName, + } + ) + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=dataset_properties + ).as_workunit() + + def _get_app_table_identifier(self, table: QlikTable) -> Optional[str]: + if table.tableQri: + return table.tableQri.lower() + return None + + def _gen_app_tables_workunit( + self, tables: List[QlikTable], app_id: str + ) -> Iterable[MetadataWorkUnit]: + for table in tables: + table_identifier = self._get_app_table_identifier(table) + if not table_identifier: + continue + dataset_urn = self._gen_qlik_dataset_urn(table_identifier) + + yield self._gen_entity_status_aspect(dataset_urn) + + yield self._gen_schema_metadata(table_identifier, table.datasetSchema) + + yield self._gen_app_table_properties(dataset_urn, table) + + yield from add_entity_to_container( + container_key=self._gen_app_key(app_id), + entity_type="dataset", + entity_urn=dataset_urn, + ) + + dpi_aspect = self._gen_dataplatform_instance_aspect(dataset_urn) + if dpi_aspect: + yield dpi_aspect + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=SubTypes(typeNames=[DatasetSubTypes.QLIK_DATASET]), + ).as_workunit() + + upstream_lineage_workunit = self._gen_app_table_upstream_lineage( + dataset_urn, table + ) + if upstream_lineage_workunit: + yield upstream_lineage_workunit + + def _gen_app_workunit(self, app: App) -> Iterable[MetadataWorkUnit]: + """ + Map Qlik App to Datahub container + """ + owner_username = self.qlik_api.get_user_name(app.ownerId) + yield from gen_containers( + container_key=self._gen_app_key(app.id), + name=app.qTitle, + description=app.description, + sub_types=[BIContainerSubTypes.QLIK_APP], + parent_container_key=self._gen_space_key(app.spaceId), + extra_properties={Constant.QRI: app.qri, Constant.USAGE: app.qUsage}, + owner_urn=builder.make_user_urn(owner_username) + if self.config.ingest_owner and owner_username + else None, + external_url=f"https://{self.config.tenant_hostname}/sense/app/{app.id}/overview", + created=int(app.createdAt.timestamp() * 1000), + last_modified=int(app.updatedAt.timestamp() * 1000), + ) + yield from self._gen_sheets_workunit(app) + yield from self._gen_app_tables_workunit(app.tables, app.id) + + def _gen_qlik_dataset_urn(self, dataset_identifier: str) -> str: + return builder.make_dataset_urn_with_platform_instance( + name=dataset_identifier, + env=self.config.env, + platform=self.platform, + platform_instance=self.config.platform_instance, + ) + + def _gen_dataplatform_instance_aspect( + self, entity_urn: str + ) -> Optional[MetadataWorkUnit]: + if self.config.platform_instance: + aspect = DataPlatformInstanceClass( + platform=builder.make_data_platform_urn(self.platform), + instance=builder.make_dataplatform_instance_urn( + self.platform, self.config.platform_instance + ), + ) + return MetadataChangeProposalWrapper( + entityUrn=entity_urn, aspect=aspect + ).as_workunit() + else: + return None + + def _gen_schema_fields( + self, schema: List[QlikDatasetSchemaField] + ) -> List[SchemaField]: + schema_fields: List[SchemaField] = [] + for field in schema: + schema_field = SchemaField( + fieldPath=field.name, + type=SchemaFieldDataTypeClass( + type=FIELD_TYPE_MAPPING.get(field.dataType, NullType)() + if field.dataType + else NullType() + ), + nativeDataType=field.dataType if field.dataType else "", + nullable=field.nullable, + isPartOfKey=field.primaryKey, + ) + schema_fields.append(schema_field) + return schema_fields + + def _gen_schema_metadata( + self, dataset_identifier: str, dataset_schema: List[QlikDatasetSchemaField] + ) -> MetadataWorkUnit: + dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) + + schema_metadata = SchemaMetadata( + schemaName=dataset_identifier, + platform=builder.make_data_platform_urn(self.platform), + version=0, + hash="", + platformSchema=MySqlDDL(tableSchema=""), + fields=self._gen_schema_fields(dataset_schema), + ) + + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=schema_metadata + ).as_workunit() + + def _gen_dataset_properties( + self, dataset_urn: str, dataset: QlikDataset + ) -> MetadataWorkUnit: + dataset_properties = DatasetProperties( + name=dataset.name, + description=dataset.description, + qualifiedName=dataset.name, + externalUrl=f"https://{self.config.tenant_hostname}/dataset/{dataset.itemId}", + created=TimeStamp(time=int(dataset.createdAt.timestamp() * 1000)), + lastModified=TimeStamp(time=int(dataset.updatedAt.timestamp() * 1000)), + ) + dataset_properties.customProperties.update( + { + Constant.QRI: dataset.secureQri, + Constant.SPACEID: dataset.spaceId, + Constant.TYPE: "Qlik Dataset", + Constant.DATASETTYPE: dataset.type, + Constant.SIZE: str(dataset.size), + Constant.ROWCOUNT: str(dataset.rowCount), + } + ) + + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=dataset_properties + ).as_workunit() + + def _get_qlik_dataset_identifier(self, dataset: QlikDataset) -> str: + return f"{dataset.spaceId}.{dataset.name}".lower() + + def _gen_dataset_workunit(self, dataset: QlikDataset) -> Iterable[MetadataWorkUnit]: + dataset_identifier = self._get_qlik_dataset_identifier(dataset) + dataset_urn = self._gen_qlik_dataset_urn(dataset_identifier) + + yield self._gen_entity_status_aspect(dataset_urn) + + yield self._gen_schema_metadata(dataset_identifier, dataset.datasetSchema) + + yield self._gen_dataset_properties(dataset_urn, dataset) + + yield from add_entity_to_container( + container_key=self._gen_space_key(dataset.spaceId), + entity_type="dataset", + entity_urn=dataset_urn, + ) + + dpi_aspect = self._gen_dataplatform_instance_aspect(dataset_urn) + if dpi_aspect: + yield dpi_aspect + + owner_username = self.qlik_api.get_user_name(dataset.ownerId) + if self.config.ingest_owner and owner_username: + yield self._gen_entity_owner_aspect(dataset_urn, owner_username) + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=SubTypes(typeNames=[DatasetSubTypes.QLIK_DATASET]), + ).as_workunit() + + def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: + return [ + *super().get_workunit_processors(), + StaleEntityRemovalHandler.create( + self, self.config, self.ctx + ).workunit_processor, + ] + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + """ + Datahub Ingestion framework invoke this method + """ + logger.info("Qlik Sense plugin execution is started") + for space in self._get_allowed_spaces(): + yield from self._gen_space_workunit(space) + for item in self.qlik_api.get_items(): + if isinstance(item, App): + yield from self._gen_app_workunit(item) + elif isinstance(item, QlikDataset): + yield from self._gen_dataset_workunit(item) + + def get_report(self) -> SourceReport: + return self.reporter diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/websocket_connection.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/websocket_connection.py new file mode 100644 index 0000000000000..01ca9415f886a --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/websocket_connection.py @@ -0,0 +1,55 @@ +import json +from typing import Dict, List, Optional, Union + +from websocket import WebSocket, create_connection + +from datahub.ingestion.source.qlik_sense.config import Constant + + +class WebsocketConnection: + def __init__(self, tenant_hostname: str, api_key: str, app_id: str) -> None: + self.websocket_url = f"wss://{tenant_hostname}/app" + self.socket_connection: Optional[WebSocket] = create_connection( + f"{self.websocket_url}/{app_id}", + header={"Authorization": f"Bearer {api_key}"}, + ) + self.request_id = 0 + self.handle = [-1] + + def _build_websocket_request_dict( + self, method: str, params: Union[Dict, List] = {} + ) -> Dict: + return { + "jsonrpc": "2.0", + "id": self.request_id, + "handle": self.handle[-1], + "method": method, + "params": params, + } + + def _send_request(self, request: Dict) -> Dict: + if self.socket_connection: + self.socket_connection.send(json.dumps(request)) + resp = self.socket_connection.recv() + if request["handle"] == -1: + resp = self.socket_connection.recv() + return json.loads(resp)[Constant.RESULT] + return {} + + def websocket_send_request( + self, method: str, params: Union[Dict, List] = {} + ) -> Dict: + """ + Method to send request to websocket + """ + self.request_id += 1 + request = self._build_websocket_request_dict(method, params) + response = self._send_request(request=request) + handle = response.get(Constant.QRETURN, {}).get(Constant.QHANDLE) + if handle: + self.handle.append(handle) + return response + + def close_websocket(self) -> None: + if self.socket_connection: + self.socket_connection.close() diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json new file mode 100644 index 0000000000000..6917b833d05a3 --- /dev/null +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json @@ -0,0 +1,1680 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-sense", + "instance": "qlik_sense_platform", + "space": "659d0e41d1b0ecce6eebc9b1", + "type": "SpaceType.SHARED" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=659d0e41d1b0ecce6eebc9b1", + "name": "test_space", + "description": "", + "created": { + "time": 1704771818002 + }, + "lastModified": { + "time": 1704771818002 + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546102, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546103, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546104, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546105, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546106, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546106, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-sense", + "instance": "qlik_sense_platform", + "space": "personal-space-id", + "type": "SpaceType.PERSONAL" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=personal-space-id", + "name": "personal_space", + "description": "", + "created": { + "time": 1707327546001 + }, + "lastModified": { + "time": 1707327546001 + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546121, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546122, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546123, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546124, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546124, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-sense", + "instance": "qlik_sense_platform", + "app": "f0714ca7-7093-49e4-8b58-47bb38563647", + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647", + "usage": "ANALYTICS" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/overview", + "name": "IPL_Matches_2022", + "description": "", + "created": { + "time": 1704803736545 + }, + "lastModified": { + "time": 1705297020333 + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546126, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546127, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546127, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik App" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546128, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546129, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + } + }, + "systemMetadata": { + "lastObserved": 1707327546130, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546130, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546132, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "chartCount": "1" + }, + "title": "New ds sheet", + "description": "", + "charts": [ + "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1705296709704, + "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" + }, + "lastModified": { + "time": 1706511226868, + "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" + } + }, + "dashboardUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/sheet/f4f57386-263a-4ec9-b40c-abcd2467f423/state/analysis" + } + }, + "systemMetadata": { + "lastObserved": 1707327546132, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + }, + "systemMetadata": { + "lastObserved": 1707327546133, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546134, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546135, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546136, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546137, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "Dimension": "[AxisProperty(Title='City', Min='NaN', Max='NaN')]", + "Measure": "[AxisProperty(Title='Sum(Date)', Min='89411', Max='2144260')]" + }, + "title": "Test_chart", + "description": "scatterplot", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707803447230, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + }, + "systemMetadata": { + "lastObserved": 1707393018679, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,qlik_sense_platform.QYUUb)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018680, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546139, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546140, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "type": "Qlik Table", + "dataconnectorid": "bb5be407-d3d3-4f19-858c-e71d593f09ae", + "dataconnectorName": "Google_BigQuery_harshal-playground-306419" + }, + "name": "test_table", + "qualifiedName": "test_table", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327546142, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + }, + "systemMetadata": { + "lastObserved": 1707327546142, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546143, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546144, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,google-cloud.harshal-playground-306419.test_dataset.test_table,DEV)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,google-cloud.harshal-playground-306419.test_dataset.test_table,DEV),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD),name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018687, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546145, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546147, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546147, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "type": "Qlik Table", + "dataconnectorid": "87d7bc7e-77d8-40dc-a251-3a35ec107b4e", + "dataconnectorName": "DataFiles" + }, + "name": "IPL_Matches_2022", + "qualifiedName": "IPL_Matches_2022", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327546149, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + }, + "systemMetadata": { + "lastObserved": 1707327546150, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546151, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546151, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD),City)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD),Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD),Date)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018696, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546153, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546155, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Season", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546155, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "type": "Qlik Dataset", + "datasetType": "DELIMETED", + "size": "38168", + "rowCount": "74" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/659d8aef12794f37026cb262", + "name": "IPL_Matches_2022.csv", + "qualifiedName": "IPL_Matches_2022.csv", + "description": "", + "created": { + "time": 1704803735527 + }, + "lastModified": { + "time": 1704804512453 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327546158, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + } + }, + "systemMetadata": { + "lastObserved": 1707327546159, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546160, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546161, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546161, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546162, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327546163, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "personal-space-id.test_tabl", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546164, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "qri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", + "spaceId": "personal-space-id", + "type": "Qlik Dataset", + "datasetType": "CONNECTION_BASED_DATASET", + "size": "0", + "rowCount": "1" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/65a137c8d5a03b02d359624a", + "name": "test_tabl", + "qualifiedName": "test_tabl", + "description": "", + "created": { + "time": 1705044592288 + }, + "lastModified": { + "time": 1705045296325 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327546166, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" + } + }, + "systemMetadata": { + "lastObserved": 1707327546167, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + }, + "systemMetadata": { + "lastObserved": 1707327546167, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327546168, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546169, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546170, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546170, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546171, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546172, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,qlik_sense_platform.f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546172, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546173, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + }, + { + "id": "urn:li:container:784b26286a16989c6329d372ccc2f97e", + "urn": "urn:li:container:784b26286a16989c6329d372ccc2f97e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546174, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f", + "urn": "urn:li:container:c2b8d174a817b41fbbd501fd4a84248f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546175, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform.personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:qlik-sense,qlik_sense_platform)" + }, + { + "id": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22", + "urn": "urn:li:container:3a7e869ba9041a0b4a4185df655b4c22" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327546176, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json new file mode 100644 index 0000000000000..5b47c09a79143 --- /dev/null +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json @@ -0,0 +1,1517 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-sense", + "space": "659d0e41d1b0ecce6eebc9b1", + "type": "SpaceType.SHARED" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=659d0e41d1b0ecce6eebc9b1", + "name": "test_space", + "description": "", + "created": { + "time": 1704771818002 + }, + "lastModified": { + "time": 1704771818002 + } + } + }, + "systemMetadata": { + "lastObserved": 1707327442983, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327442985, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense" + } + }, + "systemMetadata": { + "lastObserved": 1707327442985, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327442986, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327442987, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327442988, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-sense", + "space": "personal-space-id", + "type": "SpaceType.PERSONAL" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/catalog?space_filter=personal-space-id", + "name": "personal_space", + "description": "", + "created": { + "time": 1707327442833 + }, + "lastModified": { + "time": 1707327442833 + } + } + }, + "systemMetadata": { + "lastObserved": 1707327443007, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443008, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense" + } + }, + "systemMetadata": { + "lastObserved": 1707327443010, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443010, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443011, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "qlik-sense", + "app": "f0714ca7-7093-49e4-8b58-47bb38563647", + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647", + "usage": "ANALYTICS" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/overview", + "name": "IPL_Matches_2022", + "description": "", + "created": { + "time": 1704803736545 + }, + "lastModified": { + "time": 1705297020333 + } + } + }, + "systemMetadata": { + "lastObserved": 1707327443012, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443014, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:qlik-sense" + } + }, + "systemMetadata": { + "lastObserved": 1707327443014, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik App" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443015, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327443016, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + } + }, + "systemMetadata": { + "lastObserved": 1707327443017, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443018, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443019, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "chartCount": "1" + }, + "title": "New ds sheet", + "description": "", + "charts": [ + "urn:li:chart:(qlik-sense,QYUUb)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1705296709704, + "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" + }, + "lastModified": { + "time": 1706511226868, + "actor": "urn:li:corpuser:657b5abe656297cec3d8b205" + } + }, + "dashboardUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/sense/app/f0714ca7-7093-49e4-8b58-47bb38563647/sheet/f4f57386-263a-4ec9-b40c-abcd2467f423/state/analysis" + } + }, + "systemMetadata": { + "lastObserved": 1707327443020, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + }, + "systemMetadata": { + "lastObserved": 1707327443022, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327443023, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443024, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,QYUUb)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443025, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,QYUUb)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "Dimension": "[AxisProperty(Title='City', Min='NaN', Max='NaN')]", + "Measure": "[AxisProperty(Title='Sum(Date)', Min='89411', Max='2144260')]" + }, + "title": "Test_chart", + "description": "scatterplot", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707803447013, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,QYUUb)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + }, + "systemMetadata": { + "lastObserved": 1707393018435, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(qlik-sense,QYUUb)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018436, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443028, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443029, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "type": "Qlik Table", + "dataconnectorid": "bb5be407-d3d3-4f19-858c-e71d593f09ae", + "dataconnectorName": "Google_BigQuery_harshal-playground-306419" + }, + "name": "test_table", + "qualifiedName": "test_table", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443031, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + }, + "systemMetadata": { + "lastObserved": 1707327443032, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443033, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_dataset.test_table,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,harshal-playground-306419.test_dataset.test_table,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD),name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018442, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443035, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443037, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443038, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "type": "Qlik Table", + "dataconnectorid": "87d7bc7e-77d8-40dc-a251-3a35ec107b4e", + "dataconnectorName": "DataFiles" + }, + "name": "IPL_Matches_2022", + "qualifiedName": "IPL_Matches_2022", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443040, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + }, + "systemMetadata": { + "lastObserved": 1707327443041, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443042, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD),City)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD),Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD),Date)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707393018451, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443045, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443048, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Season", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443050, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "type": "Qlik Dataset", + "datasetType": "DELIMETED", + "size": "38168", + "rowCount": "74" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/659d8aef12794f37026cb262", + "name": "IPL_Matches_2022.csv", + "qualifiedName": "IPL_Matches_2022.csv", + "description": "", + "created": { + "time": 1704803735527 + }, + "lastModified": { + "time": 1704804512453 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443055, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + } + }, + "systemMetadata": { + "lastObserved": 1707327443057, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327443058, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443060, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443062, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1707327443065, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "personal-space-id.test_tabl", + "platform": "urn:li:dataPlatform:qlik-sense", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443066, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "qri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", + "spaceId": "personal-space-id", + "type": "Qlik Dataset", + "datasetType": "CONNECTION_BASED_DATASET", + "size": "0", + "rowCount": "1" + }, + "externalUrl": "https://iq37k6byr9lgam8.us.qlikcloud.com/dataset/65a137c8d5a03b02d359624a", + "name": "test_tabl", + "qualifiedName": "test_tabl", + "description": "", + "created": { + "time": 1705044592288 + }, + "lastModified": { + "time": 1705045296325 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443069, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + } + }, + "systemMetadata": { + "lastObserved": 1707327443071, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham jagtap", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1707327443073, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Qlik Dataset" + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443073, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443074, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443076, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1707327443076, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443078, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(qlik-sense,f4f57386-263a-4ec9-b40c-abcd2467f423)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443079, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#jokg8u7cvizvgxwrfsyxru0ykr2rl2wfd5djph9bj5q#rrg6-1cerbo4ews9o--qup3toxhm5molizgy6_wcxje,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443080, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,qri:qdf:space://ebw1euduywmui8p2bm7cor5oathzuyxvt0bircc2iru#_s2ws5rvxaarzazlmghwjhngq8znjagxptal6jlzoaw#fcj-h2tvmayi--l6fn0vqgpthf8kb2rj7sj0_ysrhgc,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + }, + { + "id": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac", + "urn": "urn:li:container:43defedfcb41b246659c449a6f3ea8ac" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443081, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,659d0e41d1b0ecce6eebc9b1.ipl_matches_2022.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704", + "urn": "urn:li:container:88cf1accecf63ec7669dc1ec7cb28704" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443082, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:qlik-sense,personal-space-id.test_tabl,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf", + "urn": "urn:li:container:1128aaac0b93f59ab99672a0b23a1bbf" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1707327443082, + "runId": "qlik-sense-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py new file mode 100644 index 0000000000000..00a50b7dbe54c --- /dev/null +++ b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py @@ -0,0 +1,1088 @@ +from typing import Any, Dict +from unittest.mock import patch + +import pytest + +from datahub.ingestion.run.pipeline import Pipeline +from tests.test_helpers import mce_helpers + + +def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: + api_vs_response: Dict[str, Dict] = { + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/api-keys": { + "method": "GET", + "status_code": 200, + "json": {}, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/spaces": { + "method": "GET", + "status_code": 200, + "json": { + "data": [ + { + "id": "659d0e41d1b0ecce6eebc9b1", + "type": "shared", + "ownerId": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "name": "test_space", + "description": "", + "meta": { + "actions": ["create", "delete", "read", "update"], + "roles": [], + "assignableRoles": [ + "codeveloper", + "consumer", + "dataconsumer", + "facilitator", + "producer", + ], + }, + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/spaces/659d0e41d1b0ecce6eebc9b1" + }, + "assignments": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/spaces/659d0e41d1b0ecce6eebc9b1/assignments" + }, + }, + "createdAt": "2024-01-09T09:13:38.002Z", + "createdBy": "657b5abe656297cec3d8b205", + "updatedAt": "2024-01-09T09:13:38.002Z", + } + ] + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items": { + "method": "GET", + "status_code": 200, + "json": { + "data": [ + { + "name": "test_app", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "thumbnailId": "", + "resourceAttributes": { + "_resourcetype": "app", + "createdDate": "2024-01-25T17:52:24.922Z", + "description": "", + "dynamicColor": "", + "hasSectionAccess": False, + "id": "f0714ca7-7093-49e4-8b58-47bb38563647", + "isDirectQueryMode": False, + "lastReloadTime": "2024-01-25T17:56:00.902Z", + "modifiedDate": "2024-01-25T17:56:02.045Z", + "name": "test_app", + "originAppId": "", + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "publishTime": "", + "published": False, + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "thumbnail": "", + "usage": "ANALYTICS", + }, + "resourceCustomAttributes": None, + "resourceUpdatedAt": "2024-01-25T17:56:02Z", + "resourceType": "app", + "resourceId": "f0714ca7-7093-49e4-8b58-47bb38563647", + "resourceCreatedAt": "2024-01-25T17:52:24Z", + "id": "65b29fd9435c545ed042a807", + "createdAt": "2024-01-25T17:52:25Z", + "updatedAt": "2024-01-25T17:56:02Z", + "creatorId": "657b5abe656297cec3d8b205", + "updaterId": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "isFavorited": False, + "collectionIds": [], + "meta": { + "isFavorited": False, + "tags": [ + {"id": "659ce561640a2affcf0d629f", "name": "test_tag"} + ], + "collections": [], + }, + "ownerId": "657b5abe656297cec3d8b205", + "resourceReloadEndTime": "2024-01-25T17:56:00Z", + "resourceReloadStatus": "ok", + "resourceSize": {"appFile": 2057, "appMemory": 78}, + "itemViews": {"total": 3, "trend": 0.3, "unique": 1}, + }, + { + "name": "IPL_Matches_2022.csv", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "resourceAttributes": { + "appType": "QIX-DF", + "dataStoreName": "DataFilesStore", + "dataStoreType": "qix-datafiles", + "qri": "qdf:qix-datafiles:ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A:sid@659d0e41d1b0ecce6eebc9b1:IPL_Matches_2022.csv", + "secureQri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "sourceSystemId": "QIX-DF_6fb35d21-24d0-474b-bf8b-536c6e9dc717", + "technicalDescription": "", + "technicalName": "IPL_Matches_2022.csv", + "type": "DELIMETED", + "version": "2", + }, + "resourceCustomAttributes": None, + "resourceUpdatedAt": "2024-01-09T18:18:32Z", + "resourceType": "dataset", + "resourceSubType": "qix-df", + "resourceId": "659d8aef92df266ef3aa5a7c", + "resourceCreatedAt": "2024-01-09T18:05:35Z", + "id": "659d8aef12794f37026cb262", + "createdAt": "2024-01-09T18:05:35Z", + "updatedAt": "2024-01-09T18:18:32Z", + "creatorId": "657b5abe656297cec3d8b205", + "updaterId": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "isFavorited": False, + "collectionIds": [], + "meta": { + "tags": [], + "collections": [], + }, + "ownerId": "657b5abe656297cec3d8b205", + "resourceReloadEndTime": "", + "resourceReloadStatus": "", + "resourceSize": {"appFile": 0, "appMemory": 0}, + "itemViews": {}, + }, + { + "name": "test_tabl", + "resourceAttributes": { + "appType": "CONNECTION_BASED_DATASET", + "dataStoreName": "gbqqri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#QgUSMctzlGjr47d2tMP35YlS93h78jVECRqtByqxPEE", + "dataStoreType": "gbq", + "qri": "a4d1b9ef-e629-4943-809a-2713aa3a5345", + "secureQri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", + "sourceSystemId": "", + "technicalDescription": "", + "technicalName": "harshal-playground-306419'.'test_dataset'.'test_table", + "type": "CONNECTION_BASED_DATASET", + "version": "2", + }, + "resourceCustomAttributes": None, + "resourceUpdatedAt": "2024-01-12T13:11:36Z", + "resourceType": "dataset", + "resourceSubType": "connection_based_dataset", + "resourceId": "65a137c849f82a37c625151b", + "resourceCreatedAt": "2024-01-12T12:59:52Z", + "id": "65a137c8d5a03b02d359624a", + "createdAt": "2024-01-12T12:59:52Z", + "updatedAt": "2024-01-12T13:11:36Z", + "creatorId": "657b5abe656297cec3d8b205", + "updaterId": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "isFavorited": False, + "collectionIds": [], + "meta": { + "tags": [], + "collections": [], + }, + "ownerId": "657b5abe656297cec3d8b205", + "resourceReloadEndTime": "", + "resourceReloadStatus": "", + "resourceSize": {"appFile": 0, "appMemory": 0}, + }, + ], + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items" + } + }, + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/users/657b5abe656297cec3d8b205": { + "method": "GET", + "status_code": 200, + "json": { + "id": "657b5abe656297cec3d8b205", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "status": "active", + "subject": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "name": "Shubham jagtap", + "email": "Shubham.Jagtap@gslab.com", + "roles": [ + "TenantAdmin", + "AnalyticsAdmin", + "AuditAdmin", + "DataAdmin", + "Developer", + "SharedSpaceCreator", + "PrivateAnalyticsContentCreator", + "DataServicesContributor", + ], + "groups": [], + "createdAt": "2023-12-14T19:42:54.417Z", + "lastUpdatedAt": "2024-01-25T06:28:22.629Z", + "created": "2023-12-14T19:42:54.417Z", + "lastUpdated": "2024-01-25T06:28:22.629Z", + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/users/657b5abe656297cec3d8b205" + } + }, + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/data-sets/659d8aef92df266ef3aa5a7c": { + "method": "GET", + "status_code": 200, + "json": { + "id": "659d8aef92df266ef3aa5a7c", + "name": "IPL_Matches_2022.csv", + "description": "", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "ownerId": "657b5abe656297cec3d8b205", + "createdTime": "2024-01-09T18:05:35.527Z", + "createdBy": "657b5abe656297cec3d8b205", + "lastModifiedTime": "2024-01-09T18:18:32.453Z", + "lastModifiedBy": "657b5abe656297cec3d8b205", + "version": 2, + "technicalDescription": "", + "technicalName": "IPL_Matches_2022.csv", + "properties": {"ModifiedByProfileService": False}, + "dataAssetInfo": { + "id": "659d8aef92df266ef3aa5a7b", + "name": "test_space", + "dataStoreInfo": { + "id": "659d8aef92df266ef3aa5a7a", + "name": "DataFilesStore", + "type": "qix-datafiles", + }, + }, + "operational": { + "size": 38168, + "rowCount": 74, + "lastLoadTime": "2024-01-09T18:05:35.527Z", + "logMessage": '{\n "cloudEventsVersion": "0.1",\n "source": "com.qlik/qix-datafiles",\n "contentType": "application/json",\n "eventID": "48c935d3-499c-4333-96e6-c8922af6e238",\n "eventType": "com.qlik.datafile.created",\n "eventTypeVersion": "0.0.1",\n "eventTime": "2024-01-09T18:05:35.834061Z",\n "extensions": {\n "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A",\n "userId": "657b5abe656297cec3d8b205",\n "header": {\n "traceparent": [\n "00-000000000000000063d931098d8f607c-7f2bea16c07c1c3a-01"\n ]\n }\n },\n "data": {\n "id": "6fb35d21-24d0-474b-bf8b-536c6e9dc717",\n "name": "IPL_Matches_2022.csv",\n "createdDate": "2024-01-09T18:05:35.5279392Z",\n "modifiedDate": "2024-01-09T18:05:35.828813Z",\n "createdByUser": "657b5abe656297cec3d8b205",\n "modifiedByUser": "657b5abe656297cec3d8b205",\n "ownerId": "657b5abe656297cec3d8b205",\n "spaceId": "659d0e41d1b0ecce6eebc9b1",\n "size": 38168,\n "contentUpdated": true,\n "isInternal": false,\n "qri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4"\n },\n "messageTopic": "system-events.datafiles/Created/ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A/6fb35d21-24d0-474b-bf8b-536c6e9dc717"\n}', + "status": "com.qlik.datafile.created", + "location": "48c935d3-499c-4333-96e6-c8922af6e238", + "contentUpdated": True, + "lastUpdateTime": "2024-01-09T18:05:35.828Z", + }, + "schema": { + "dataFields": [ + { + "name": "ID", + "index": 0, + "dataType": { + "type": "INTEGER", + "properties": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$integer", "$numeric"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + }, + { + "name": "City", + "index": 1, + "dataType": { + "type": "STRING", + "properties": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$text", "$ascii"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + }, + { + "name": "Date", + "index": 2, + "dataType": { + "type": "DATE", + "properties": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$integer", "$numeric", "$date", "$timestamp"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + }, + { + "name": "Season", + "index": 3, + "dataType": { + "type": "INTEGER", + "properties": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$integer", "$numeric"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + }, + ], + "loadOptions": { + "qDataFormat": { + "qType": "CSV", + "qLabel": "embedded labels", + "qQuote": "msq", + "qDelimiter": { + "qName": "Comma", + "qScriptCode": "','", + "qNumber": 44, + }, + "qCodePage": 28591, + "qHeaderSize": 0, + "qRecordSize": 0, + "qTabSize": 0, + } + }, + "effectiveDate": "2024-01-09T18:05:39.713Z", + "overrideSchemaAnomalies": False, + }, + "qri": "qdf:qix-datafiles:ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A:sid@659d0e41d1b0ecce6eebc9b1:IPL_Matches_2022.csv", + "secureQri": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#cfl7k2dSDQT8_iAQ36wboHaodaoJTC9bE-sc7ZPM6q4", + "type": "DELIMETED", + "classifications": { + "personalInformation": [], + "sensitiveInformation": [], + }, + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/data-sets/65a137c849f82a37c625151b": { + "method": "GET", + "status_code": 200, + "json": { + "id": "65a137c849f82a37c625151b", + "name": "test_tabl", + "description": "", + "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", + "ownerId": "657b5abe656297cec3d8b205", + "createdTime": "2024-01-12T12:59:52.288Z", + "createdBy": "657b5abe656297cec3d8b205", + "lastModifiedTime": "2024-01-12T13:11:36.325Z", + "lastModifiedBy": "657b5abe656297cec3d8b205", + "version": 2, + "technicalDescription": "", + "technicalName": "harshal-playground-306419'.'test_dataset'.'test_table", + "properties": {"ModifiedByProfileService": False}, + "dataAssetInfo": { + "id": "65a137c849f82a37c625151a", + "name": "gbqqri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#QgUSMctzlGjr47d2tMP35YlS93h78jVECRqtByqxPEE", + "dataStoreInfo": { + "id": "65a137c849f82a37c6251519", + "name": "gbqqri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#QgUSMctzlGjr47d2tMP35YlS93h78jVECRqtByqxPEE", + "type": "gbq", + }, + }, + "operational": { + "rowCount": 1, + "contentUpdated": False, + "tableConnectionInfo": { + "tableName": "test_table", + "selectionScript": "[test_table]:\nSELECT name\nFROM `harshal-playground-306419`.`test_dataset`.`test_table`;", + "additionalProperties": { + "fields": '[{"name":"name","fullName":"name","nativeType":"STRING","nativeFieldInfo":{"dataType":12,"name":"name","nullable":1,"ordinalPostion":1,"scale":0,"size":65535,"typeName":"STRING"},"customProperties":[],"isSelected":true}]', + "tableRequestParameters": '[{\\"name\\":\\"database\\",\\"value\\":\\"harshal-playground-306419\\"},{\\"name\\":\\"owner\\",\\"value\\":\\"test_dataset\\"}]', + }, + }, + }, + "schema": { + "dataFields": [ + { + "name": "name", + "index": 0, + "dataType": { + "type": "STRING", + "properties": {"qType": "A", "qnDec": 0, "qUseThou": 0}, + }, + "tags": ["$text", "$ascii"], + "encrypted": False, + "primaryKey": False, + "sensitive": False, + "orphan": False, + "nullable": False, + } + ], + "loadOptions": {}, + "effectiveDate": "2024-01-12T12:59:54.522Z", + "anomalies": ["$warning-single-text-column"], + "overrideSchemaAnomalies": False, + }, + "qri": "a4d1b9ef-e629-4943-809a-2713aa3a5345", + "secureQri": "qri:db:gbq://DU8iuDBF_ZFeSt2FN0x_P7ypp18VBc7gYNFAgDFVoY8#Nkh5KRvqTwByLVl5up594IBQ2QA99-6N16Ux_O4qUBs", + "type": "CONNECTION_BASED_DATASET", + "classifications": { + "personalInformation": [], + "sensitiveInformation": [], + }, + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/actions/expand?node=qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647&level=TABLE": { + "method": "GET", + "status_code": 200, + "json": { + "graph": { + "id": "", + "directed": True, + "type": "TABLE", + "label": "Expansion for qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647", + "nodes": { + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc": { + "label": "IPL_Matches_2022", + "metadata": {"fields": 1, "type": "TABLE"}, + }, + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE": { + "label": "test_table", + "metadata": {"fields": 1, "type": "TABLE"}, + }, + }, + "edges": [ + { + "relation": "from", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#t91dFUED_ebvKHOjauxSOOCnlRZQTwEkNHv_bjUl7AY#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + }, + ], + "metadata": {}, + } + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/actions/expand?node=qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647%23FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc&level=FIELD": { + "method": "GET", + "status_code": 200, + "json": { + "graph": { + "id": "", + "directed": True, + "type": "FIELD", + "label": "Expansion for qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + "nodes": { + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#-NAdo6895jVLliOXp13Do7e1kDNDDPwxZ0CDuGFJb8A": { + "label": "City", + "metadata": {"type": "FIELD"}, + }, + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#3DAdoo7e1kDNDDPwxZ0CD3Do7e1kDNDDPwxZ0CDuGFJ": { + "label": "Date", + "metadata": {"type": "FIELD"}, + }, + }, + "edges": [ + { + "relation": "from", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#fpA_oxHhnvRu-RrTC_INPo-TEykis1Y_6e-Qy0rKy2I#ti1Pyd82WD92n5Mafl23Gs1Pr_pwJ1tyFIWtaTJfslc#BcCb_mytkkVA-HSHbFLRYOHFY-O_55m8X_JG7DNzMNE", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#-NAdo6895jVLliOXp13Do7e1kDNDDPwxZ0CDuGFJb8A", + }, + { + "relation": "from", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#fpA_oxHhnvRu-RrTC_INPo-TEykis1Y_6e-Qy0rKy2I#ti1Pyd82WD92n5Mafl23Gs1Pr_pwJ1tyFIWtaTJfslc#BcCb_mytkkVA-HSHbFLRYOHFY-O_58X8X_JG7DNzMNE", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#3DAdoo7e1kDNDDPwxZ0CD3Do7e1kDNDDPwxZ0CDuGFJ", + }, + ], + "metadata": {}, + } + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/actions/expand?node=qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647%23Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE&level=FIELD": { + "method": "GET", + "status_code": 200, + "json": { + "graph": { + "id": "", + "directed": True, + "type": "FIELD", + "label": "Expansion for qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE", + "nodes": { + "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok": { + "label": "name", + "metadata": {"type": "FIELD"}, + } + }, + "edges": [ + { + "relation": "read", + "source": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#JOKG8u7CvizvGXwrFsyXRU0yKr2rL2WFD5djpH9bj5Q#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok", + "target": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok", + } + ], + "metadata": {}, + } + }, + }, + "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/lineage-graphs/nodes/qri%3Aapp%3Asense%3A%2F%2Ff0714ca7-7093-49e4-8b58-47bb38563647/overview": { + "method": "POST", + "response_list": [ + { + "status_code": 200, + "json": { + "resources": [ + { + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc#-NAdo6895jVLliOXp13Do7e1kDNDDPwxZ0CDuGFJb8A", + "lineage": [ + { + "resourceLabel": "d7a6c03238b0c25b3d5c8631e1121807.qvd", + "resourceQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#_s2Ws5RVxAArzAZlmghwjHNGq8ZnjagxptAl6jlZOaw", + "tableQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#_s2Ws5RVxAArzAZlmghwjHNGq8ZnjagxptAl6jlZOaw#FcJ-H2TvmAyI--l6fn0VQGPtHf8kB2rj7sj0_ysRHgc", + "tableLabel": "IPL_Matches_2022", + }, + ], + } + ] + }, + }, + { + "status_code": 200, + "json": { + "resources": [ + { + "qri": "qri:app:sense://f0714ca7-7093-49e4-8b58-47bb38563647#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE#gqNTf_Dbzn7sNdae3DoYnubxfYLzU6VT-aqWywvjzok", + "lineage": [ + { + "resourceLabel": "3fcbce17de9e55774ddedc345532c419.qvd", + "resourceQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#JOKG8u7CvizvGXwrFsyXRU0yKr2rL2WFD5djpH9bj5Q", + "tableQRI": "qri:qdf:space://Ebw1EudUywmUi8p2bM7COr5OATHzuYxvT0BIrCc2irU#JOKG8u7CvizvGXwrFsyXRU0yKr2rL2WFD5djpH9bj5Q#Rrg6-1CeRbo4ews9o--QUP3tOXhm5moLizGY6_wCxJE", + "tableLabel": "test_table", + } + ], + } + ] + }, + }, + ], + }, + } + + api_vs_response.update(override_data) + + for url in api_vs_response.keys(): + if api_vs_response[url].get("response_list"): + request_mock.register_uri( + api_vs_response[url]["method"], + url, + response_list=api_vs_response[url]["response_list"], + ) + else: + request_mock.register_uri( + api_vs_response[url]["method"], + url, + json=api_vs_response[url]["json"], + status_code=api_vs_response[url]["status_code"], + ) + + +def mock_websocket_response(*args, **kwargs): + request = kwargs["request"] + if request == { + "jsonrpc": "2.0", + "id": 1, + "handle": -1, + "method": "OpenDoc", + "params": {"qDocName": "f0714ca7-7093-49e4-8b58-47bb38563647"}, + }: + return { + "qReturn": { + "qType": "Doc", + "qHandle": 1, + "qGenericId": "f0714ca7-7093-49e4-8b58-47bb38563647", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 2, + "handle": 1, + "method": "GetAppLayout", + "params": {}, + }: + return { + "qLayout": { + "qTitle": "IPL_Matches_2022", + "qFileName": "f0714ca7-7093-49e4-8b58-47bb38563647", + "qLastReloadTime": "2024-01-15T11:06:53.070Z", + "qHasScript": True, + "qStateNames": [], + "qMeta": {}, + "qHasData": True, + "qThumbnail": {}, + "qUnsupportedFeatures": [], + "qUsage": "ANALYTICS", + "encrypted": True, + "id": "f0714ca7-7093-49e4-8b58-47bb38563647", + "published": False, + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "createdDate": "2024-01-09T18:05:36.545Z", + "modifiedDate": "2024-01-15T11:07:00.333Z", + "spaceId": "659d0e41d1b0ecce6eebc9b1", + "hassectionaccess": False, + "_resourcetype": "app", + "privileges": [ + "read", + "update", + "delete", + "reload", + "export", + "duplicate", + "change_owner", + "change_space", + "export_reduced", + "source", + "exportappdata", + ], + } + } + elif request == { + "jsonrpc": "2.0", + "id": 3, + "handle": 1, + "method": "GetObjects", + "params": {"qOptions": {"qTypes": ["sheet"]}}, + }: + return { + "qList": [ + { + "qInfo": { + "qId": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "qType": "sheet", + }, + "qMeta": { + "title": "New ds sheet", + "description": "", + "_resourcetype": "app.object", + "_objecttype": "sheet", + "id": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "approved": False, + "published": False, + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "createdDate": "2024-01-15T11:01:49.704Z", + "modifiedDate": "2024-01-29T12:23:46.868Z", + "publishTime": True, + "privileges": [ + "read", + "update", + "delete", + "publish", + "change_owner", + ], + }, + "qData": {}, + }, + ] + } + elif request == { + "jsonrpc": "2.0", + "id": 4, + "handle": 1, + "method": "GetObject", + "params": {"qId": "f4f57386-263a-4ec9-b40c-abcd2467f423"}, + }: + return { + "qReturn": { + "qType": "GenericObject", + "qHandle": 2, + "qGenericType": "sheet", + "qGenericId": "f4f57386-263a-4ec9-b40c-abcd2467f423", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 5, + "handle": 2, + "method": "GetLayout", + "params": {}, + }: + return { + "qLayout": { + "qInfo": { + "qId": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "qType": "sheet", + }, + "qMeta": { + "title": "New ds sheet", + "description": "", + "_resourcetype": "app.object", + "_objecttype": "sheet", + "id": "f4f57386-263a-4ec9-b40c-abcd2467f423", + "approved": False, + "published": False, + "owner": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", + "ownerId": "657b5abe656297cec3d8b205", + "createdDate": "2024-01-15T11:01:49.704Z", + "modifiedDate": "2024-01-29T12:23:46.868Z", + "publishTime": True, + "privileges": [ + "read", + "update", + "delete", + "publish", + "change_owner", + ], + }, + "qSelectionInfo": {}, + "rank": 0, + "thumbnail": {"qStaticContentUrl": {}}, + "columns": 24, + "rows": 12, + "qChildList": { + "qItems": [ + { + "qInfo": {"qId": "QYUUb", "qType": "barchart"}, + "qMeta": {"privileges": ["read", "update", "delete"]}, + "qData": {"title": ""}, + } + ] + }, + "customRowBase": 12, + "gridResolution": "small", + "layoutOptions": {"mobileLayout": "LIST", "extendable": False}, + "gridMode": "simpleEdit", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 6, + "handle": 2, + "method": "GetChild", + "params": {"qId": "QYUUb"}, + }: + return { + "qReturn": { + "qType": "GenericObject", + "qHandle": 3, + "qGenericType": "scatterplot", + "qGenericId": "QYUUb", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 7, + "handle": 3, + "method": "GetLayout", + "params": {}, + }: + return { + "qLayout": { + "qInfo": {"qId": "QYUUb", "qType": "scatterplot"}, + "qMeta": {"privileges": ["read", "update", "delete"]}, + "qSelectionInfo": {}, + "qHyperCube": { + "qSize": {"qcx": 3, "qcy": 5}, + "qDimensionInfo": [ + { + "qFallbackTitle": "City", + "qApprMaxGlyphCount": 11, + "qCardinal": 5, + "qSortIndicator": "A", + "qGroupFallbackTitles": ["City"], + "qGroupPos": 0, + "qStateCounts": { + "qLocked": 0, + "qSelected": 0, + "qOption": 5, + "qDeselected": 0, + "qAlternative": 0, + "qExcluded": 0, + "qSelectedExcluded": 0, + "qLockedExcluded": 0, + }, + "qTags": [ + "$ascii", + "$text", + "$geoname", + "$relates_IPL_Matches_2022.City_GeoInfo", + ], + "qDimensionType": "D", + "qGrouping": "N", + "qNumFormat": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + "qIsAutoFormat": True, + "qGroupFieldDefs": ["City"], + "qMin": "NaN", + "qMax": "NaN", + "qAttrExprInfo": [], + "qAttrDimInfo": [], + "qCardinalities": { + "qCardinal": 5, + "qHypercubeCardinal": 5, + "qAllValuesCardinal": -1, + }, + "autoSort": True, + "cId": "FdErA", + "othersLabel": "Others", + } + ], + "qMeasureInfo": [ + { + "qFallbackTitle": "Sum(Date)", + "qApprMaxGlyphCount": 7, + "qCardinal": 0, + "qSortIndicator": "D", + "qNumFormat": {"qType": "U", "qnDec": 0, "qUseThou": 0}, + "qMin": 89411, + "qMax": 2144260, + "qIsAutoFormat": True, + "qAttrExprInfo": [], + "qAttrDimInfo": [], + "qTrendLines": [], + "autoSort": True, + "cId": "PtTpyG", + "numFormatFromTemplate": True, + }, + ], + }, + "script": "", + "showTitles": True, + "title": "Test_chart", + "subtitle": "", + "footnote": "", + "disableNavMenu": False, + "showDetails": True, + "showDetailsExpression": False, + "visualization": "scatterplot", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 8, + "handle": 1, + "method": "GetObject", + "params": ["LoadModel"], + }: + return { + "qReturn": { + "qType": "GenericObject", + "qHandle": 4, + "qGenericType": "LoadModel", + "qGenericId": "LoadModel", + } + } + elif request == { + "jsonrpc": "2.0", + "id": 9, + "handle": 4, + "method": "GetLayout", + "params": {}, + }: + return { + "qLayout": { + "qInfo": {"qId": "LoadModel", "qType": "LoadModel"}, + "tables": [ + { + "dataconnectorName": "Google_BigQuery_harshal-playground-306419", + "dataconnectorPrefix": "test_space:", + "boxType": "blackbox", + "databaseName": "", + "ownerName": "", + "tableName": "test_table", + "tableAlias": "test_table", + "loadProperties": { + "filterInfo": {"filterClause": "", "filterType": 1} + }, + "key": "Google_BigQuery_harshal-playground-306419:::test_table", + "fields": [ + { + "alias": "name", + "name": "name", + "selected": True, + "checked": True, + "id": "dsd.test_table.name", + } + ], + "connectionInfo": { + "name": "Google_BigQuery_harshal-playground-306419", + "displayName": "Google_BigQuery_harshal-playground-306419", + "id": "bb5be407-d3d3-4f19-858c-e71d593f09ae", + "type": { + "provider": "QvOdbcConnectorPackage.exe", + "type": "custom", + "name": "QvOdbcConnectorPackage", + "displayName": "Qlik® ODBC Connector Package", + "isStandardConnector": False, + "isIframeCompatible": True, + "needsConnect": True, + "connectDialog": "/customdata/64/QvOdbcConnectorPackage/web/standalone/connect-dialog.html?locale=en-US", + "selectDialog": "/customdata/64/QvOdbcConnectorPackage/web/standalone/select-dialog.html?locale=en-US", + "selectAddData": "/customdata/64/QvOdbcConnectorPackage/web/standalone/select-adddata.html?locale=en-US", + "credentialsDialog": "/customdata/64/QvOdbcConnectorPackage/web/standalone/credentials-dialog.html?locale=en-US", + "update": "/customdata/64/QvOdbcConnectorPackage/web/standalone/loadModelUpdate.js", + "architecture": {"text": "Common.undefinedbit"}, + "connectorMain": "QvOdbcConnectorPackage.webroot/connector-main-iframe", + }, + "typeName": "QvOdbcConnectorPackage.exe", + "privileges": [ + "change_owner", + "change_space", + "delete", + "list", + "read", + "update", + ], + "sourceConnectorID": "gbq", + "dataconnectorPrefix": "test_space:", + "isInAppSpace": True, + "space": "659d0e41d1b0ecce6eebc9b1", + "connectionString": 'CUSTOM CONNECT TO "provider=QvOdbcConnectorPackage.exe;driver=gbq;OAuthMechanism=0;SupportOldClient=true;Catalog_Old=harshal-playground-306419;separateCredentials=false;Catalog=harshal-playground-306419;Min_TLS=1.2;SQLDialect=1;RowsFetchedPerBlock=16384;DefaultStringColumnLength=65535;AllowLargeResults=false;EnableHTAPI=false;HTAPI_MinResultsSize=1000;HTAPI_MinActivationRatio=3;allowNonSelectQueries=false;QueryTimeout=30;Timeout=300;useBulkReader=true;bulkFetchSize=50;rowBatchSize=1;bulkFetchColumnMode=true;maxStringLength=4096;logSQLStatements=false;"', + "hasEditableSeparatedCredentials": False, + "canDelete": True, + "connectorImagePath": "https://iq37k6byr9lgam8.us.qlikcloud.com/customdata/64/QvOdbcConnectorPackage/web/gbq-square.png", + "connectorDisplayName": "Google BigQuery", + "dbInfo": {}, + }, + "id": "dsd.test_table", + "tableGroupId": "", + "connectorProperties": { + "tableQualifiers": [ + "harshal-playground-306419", + "test_dataset", + ], + }, + "selectStatement": "SELECT name\nFROM `harshal-playground-306419`.`test_dataset`.`test_table`;", + "caching": {"enabled": True, "type": "qvd"}, + }, + { + "dataconnectorName": "DataFiles", + "dataconnectorPrefix": "test_space:", + "boxType": "load-file", + "databaseName": "IPL_Matches_2022.csv", + "ownerName": "TYPE9_CSV", + "tableName": "IPL_Matches_2022", + "tableAlias": "IPL_Matches_2022", + "key": "DataFiles:IPL_Matches_2022.csv:TYPE9_CSV:IPL_Matches_2022", + "fields": [ + { + "id": "dsd.IPL_Matches_2022.City", + "name": "City", + "alias": "City", + "selected": True, + }, + { + "id": "dsd.IPL_Matches_2022.Date", + "name": "Date", + "alias": "Date", + "selected": True, + }, + ], + "connectionInfo": { + "type": {"isStorageProvider": False}, + "id": "87d7bc7e-77d8-40dc-a251-3a35ec107b4e", + "name": "DataFiles", + "typeName": "qix-datafiles.exe", + "sourceConnectorID": "qix-datafiles.exe", + "connectionString": 'CUSTOM CONNECT TO "provider=qix-datafiles.exe;path=test_space:datafiles;"', + "space": "659d0e41d1b0ecce6eebc9b1", + "dataconnectorPrefix": "test_space:", + "caching": {"enabled": True, "type": "qvd"}, + }, + "id": "dsd.IPL_Matches_2022", + "loadSelectStatement": "LOAD [ID] AS [ID],\n\t[City] AS [City],\n\t[Date] AS [Date],\n\t[Season] AS [Season],\n\t[MatchNumber] AS [MatchNumber],\n\t[Team1] AS [Team1],\n\t[Team2] AS [Team2],\n\t[Venue] AS [Venue],\n\t[TossWinner] AS [TossWinner],\n\t[TossDecision] AS [TossDecision],\n\t[SuperOver] AS [SuperOver],\n\t[WinningTeam] AS [WinningTeam],\n\t[WonBy] AS [WonBy],\n\t[Margin] AS [Margin],\n\t[method] AS [method],\n\t[Player_of_Match] AS [Player_of_Match],\n\t[Team1Players] AS [Team1Players],\n\t[Team2Players] AS [Team2Players],\n\t[Umpire1] AS [Umpire1],\n\t[Umpire2] AS [Umpire2]\nFROM [lib://DataFiles/IPL_Matches_2022.csv]\n(txt, codepage is 28591, embedded labels, delimiter is ',', msq);\n", + "formatSpec": "(txt, codepage is 28591, embedded labels, delimiter is ',', msq)", + "caching": {"enabled": True, "type": "qvd"}, + }, + ], + "schemaVersion": 2.1, + } + } + else: + return {} + + +@pytest.fixture(scope="module") +def mock_websocket_send_request(): + with patch( + "datahub.ingestion.source.qlik_sense.qlik_api.WebsocketConnection._send_request" + ) as mock_websocket_send_request, patch( + "datahub.ingestion.source.qlik_sense.websocket_connection.create_connection" + ): + mock_websocket_send_request.side_effect = mock_websocket_response + yield mock_websocket_send_request + + +def default_config(): + return { + "tenant_hostname": "iq37k6byr9lgam8.us.qlikcloud.com", + "api_key": "qlik-api-key", + "space_pattern": {"allow": ["test_space", "personal_space"]}, + } + + +@pytest.mark.integration +def test_qlik_sense_ingest( + pytestconfig, tmp_path, requests_mock, mock_websocket_send_request +): + + test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_sense" + + register_mock_api(request_mock=requests_mock) + + output_path: str = f"{tmp_path}/qlik_sense_ingest_mces.json" + + pipeline = Pipeline.create( + { + "run_id": "qlik-sense-test", + "source": { + "type": "qlik-sense", + "config": { + **default_config(), + }, + }, + "sink": { + "type": "file", + "config": { + "filename": output_path, + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_qlik_sense_ingest.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_path, + golden_path=f"{test_resources_dir}/{golden_file}", + ) + + +@pytest.mark.integration +def test_platform_instance_ingest( + pytestconfig, tmp_path, requests_mock, mock_websocket_send_request +): + + test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_sense" + + register_mock_api(request_mock=requests_mock) + + output_path: str = f"{tmp_path}/qlik_platform_instace_ingest_mces.json" + + pipeline = Pipeline.create( + { + "run_id": "qlik-sense-test", + "source": { + "type": "qlik-sense", + "config": { + **default_config(), + "platform_instance": "qlik_sense_platform", + "data_connection_to_platform_instance": { + "Google_BigQuery_harshal-playground-306419": { + "platform_instance": "google-cloud", + "env": "DEV", + } + }, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": output_path, + }, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_platform_instance_ingest.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_path, + golden_path=f"{test_resources_dir}/{golden_file}", + ) diff --git a/metadata-service/war/src/main/resources/boot/data_platforms.json b/metadata-service/war/src/main/resources/boot/data_platforms.json index 5bff689e0a555..6ef8bbc654585 100644 --- a/metadata-service/war/src/main/resources/boot/data_platforms.json +++ b/metadata-service/war/src/main/resources/boot/data_platforms.json @@ -605,6 +605,16 @@ "logoUrl": "/assets/platforms/csv-logo.png" } }, + { + "urn": "urn:li:dataPlatform:qlik-sense", + "aspect": { + "datasetNameDelimiter": ".", + "name": "qlik-sense", + "displayName": "Qlik Sense", + "type": "OTHERS", + "logoUrl": "/assets/platforms/qliklogo.png" + } + }, { "urn": "urn:li:dataPlatform:file", "aspect": {