forked from datahub-project/datahub
-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(ingest/qlik): Qlik cloud connector integration (datahub-project#…
…9682) Co-authored-by: Harshal Sheth <[email protected]>
- Loading branch information
1 parent
93acb82
commit a1f2216
Showing
17 changed files
with
5,682 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
23 changes: 23 additions & 0 deletions
23
metadata-ingestion/docs/sources/qlik-sense/qlik-sense_pre.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| ## Integration Details | ||
|
|
||
| This source extracts the following: | ||
|
|
||
| - Accessible spaces and apps within that spaces as Container. | ||
| - Qlik Datasets as Datahub Datasets with schema metadata. | ||
| - Sheets as Datahub dashboard and charts present inside sheets. | ||
|
|
||
| ## Configuration Notes | ||
|
|
||
| 1. Refer [doc](https://qlik.dev/authenticate/api-key/generate-your-first-api-key/) to generate an API key from the hub. | ||
| 2. Get tenant hostname from About tab after login to qlik sense account. | ||
|
|
||
| ## Concept mapping | ||
|
|
||
| | Qlik Sense | Datahub | Notes | | ||
| |------------------------|---------------------------------------------------------------|----------------------------------| | ||
| | `Space` | [Container](../../metamodel/entities/container.md) | SubType `"Qlik Space"` | | ||
| | `App` | [Container](../../metamodel/entities/container.md) | SubType `"Qlik App"` | | ||
| | `Sheet` | [Dashboard](../../metamodel/entities/dashboard.md) | | | ||
| | `Chart` | [Chart](../../metamodel/entities/chart.md) | | | ||
| | `Dataset` | [Dataset](../../metamodel/entities/dataset.md) | SubType `"Qlik Dataset"` | | ||
| | `User` | [User (a.k.a CorpUser)](../../metamodel/entities/corpuser.md) | Optionally Extracted | |
25 changes: 25 additions & 0 deletions
25
metadata-ingestion/docs/sources/qlik-sense/qlik-sense_recipe.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| source: | ||
| type: qlik-sense | ||
| config: | ||
| # Coordinates | ||
| tenant_hostname: "xyz12xz.us.qlikcloud.com" | ||
| # Credentials | ||
| api_key: "QLIK_API_KEY" | ||
|
|
||
| # Optional - filter for certain space names instead of ingesting everything. | ||
| # Mention 'personal_space' if entities of personal space need to ingest | ||
| # space_pattern: | ||
| # allow: | ||
| # - space_name | ||
|
|
||
| ingest_owner: true | ||
|
|
||
| # Optional -- This mapping is optional and only required to configure platform-instance for Qlik app dataset upstream source tables | ||
| # A mapping of the Qlik app dataset upstream tables from data connection to platform instance. Use 'data_connection_name' as key. | ||
| # data_connection_to_platform_instance: | ||
| # data_connection_name: | ||
| # platform_instance: cloud_instance | ||
| # env: DEV | ||
|
|
||
| sink: | ||
| # sink configs |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
130 changes: 130 additions & 0 deletions
130
metadata-ingestion/src/datahub/ingestion/source/qlik_sense/config.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| import logging | ||
| from dataclasses import dataclass | ||
| from typing import Dict, Optional | ||
|
|
||
| import pydantic | ||
|
|
||
| from datahub.configuration.common import AllowDenyPattern | ||
| from datahub.configuration.source_common import ( | ||
| EnvConfigMixin, | ||
| PlatformInstanceConfigMixin, | ||
| ) | ||
| from datahub.ingestion.source.state.stale_entity_removal_handler import ( | ||
| StaleEntityRemovalSourceReport, | ||
| ) | ||
| from datahub.ingestion.source.state.stateful_ingestion_base import ( | ||
| StatefulIngestionConfigBase, | ||
| ) | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
| QLIK_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" | ||
|
|
||
|
|
||
| class Constant: | ||
| """ | ||
| keys used in qlik plugin | ||
| """ | ||
|
|
||
| # Rest API response key constants | ||
| DATA = "data" | ||
| ID = "id" | ||
| NAME = "name" | ||
| TYPE = "type" | ||
| ITEMID = "itemId" | ||
| NEXT = "next" | ||
| LINKS = "links" | ||
| HREF = "href" | ||
| DATASETTYPE = "datasetType" | ||
| CREATEDAT = "createdAt" | ||
| UPDATEDAT = "updatedAt" | ||
| SECUREQRI = "secureQri" | ||
| QRI = "qri" | ||
| SPACEID = "spaceId" | ||
| SPACE = "space" | ||
| CREATEDTIME = "createdTime" | ||
| LASTMODIFIEDTIME = "lastModifiedTime" | ||
| OPERATIONAL = "operational" | ||
| SIZE = "size" | ||
| ROWCOUNT = "rowCount" | ||
| DATATYPE = "dataType" | ||
| PRIMARYKEY = "primaryKey" | ||
| NULLABLE = "nullable" | ||
| SCHEMA = "schema" | ||
| DATAFIELDS = "dataFields" | ||
| RESOURCETYPE = "resourceType" | ||
| USAGE = "usage" | ||
| CREATEDDATE = "createdDate" | ||
| MODIFIEDDATE = "modifiedDate" | ||
| RESOURCEID = "resourceId" | ||
| DATASETSCHEMA = "datasetSchema" | ||
| GRAPH = "graph" | ||
| NODES = "nodes" | ||
| RESOURCES = "resources" | ||
| LINEAGE = "lineage" | ||
| TABLELABEL = "tableLabel" | ||
| TABLEQRI = "tableQRI" | ||
| OWNERID = "ownerId" | ||
| # Websocket response key constants | ||
| QID = "qId" | ||
| RESULT = "result" | ||
| QRETURN = "qReturn" | ||
| QTYPE = "qType" | ||
| QHANDLE = "qHandle" | ||
| QLAYOUT = "qLayout" | ||
| QMETA = "qMeta" | ||
| QCHILDLIST = "qChildList" | ||
| QITEMS = "qItems" | ||
| QINFO = "qInfo" | ||
| QLIST = "qList" | ||
| CONNECTORPROPERTIES = "connectorProperties" | ||
| TABLEQUALIFIERS = "tableQualifiers" | ||
| CONNECTIONINFO = "connectionInfo" | ||
| SOURCECONNECTORID = "sourceConnectorID" | ||
| DATABASENAME = "databaseName" | ||
| SCHEMANAME = "schemaName" | ||
| TABLES = "tables" | ||
| DATACONNECTORID = "dataconnectorid" | ||
| DATACONNECTORNAME = "dataconnectorName" | ||
| DATACONNECTORPLATFORM = "dataconnectorPlatform" | ||
| # Item type | ||
| APP = "app" | ||
| DATASET = "dataset" | ||
| # Personal entity constants | ||
| PERSONAL_SPACE_ID = "personal-space-id" | ||
| PERSONAL_SPACE_NAME = "personal_space" | ||
|
|
||
|
|
||
| @dataclass | ||
| class QlikSourceReport(StaleEntityRemovalSourceReport): | ||
| number_of_spaces: int = 0 | ||
|
|
||
| def report_number_of_spaces(self, number_of_spaces: int) -> None: | ||
| self.number_of_spaces = number_of_spaces | ||
|
|
||
|
|
||
| class PlatformDetail(PlatformInstanceConfigMixin, EnvConfigMixin): | ||
| pass | ||
|
|
||
|
|
||
| class QlikSourceConfig( | ||
| StatefulIngestionConfigBase, PlatformInstanceConfigMixin, EnvConfigMixin | ||
| ): | ||
| tenant_hostname: str = pydantic.Field(description="Qlik Tenant hostname") | ||
| api_key: str = pydantic.Field(description="Qlik API Key") | ||
| # Qlik space identifier | ||
| space_pattern: AllowDenyPattern = pydantic.Field( | ||
| default=AllowDenyPattern.allow_all(), | ||
| description="Regex patterns to filter Qlik spaces in ingestion." | ||
| "Mention 'personal_space' if entities of personal space need to ingest", | ||
| ) | ||
| ingest_owner: Optional[bool] = pydantic.Field( | ||
| default=True, | ||
| description="Ingest Owner from source. This will override Owner info entered from UI", | ||
| ) | ||
| # Qlik app dataset upstream tables from data connection to platform instance mapping | ||
| data_connection_to_platform_instance: Dict[str, PlatformDetail] = pydantic.Field( | ||
| default={}, | ||
| description="A mapping of the Qlik app dataset upstream tables from data connection to platform instance." | ||
| "Use 'data_connection_name' as key.", | ||
| ) |
Oops, something went wrong.