Skip to content

Commit

Permalink
Merge branch 'datahub-project:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@hsheth2
hsheth2 authored Dec 23, 2024
2 parents 5d42a91 + 21ddb55 commit 5ec0ec9
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 9 deletions.
21 changes: 14 additions & 7 deletions metadata-ingestion/src/datahub/ingestion/source/mode.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@
TagPropertiesClass,
UpstreamClass,
UpstreamLineageClass,
ViewPropertiesClass,
)
from datahub.metadata.urns import QueryUrn
from datahub.sql_parsing.sqlglot_lineage import (
Expand Down Expand Up @@ -930,16 +931,13 @@ def construct_query_or_dataset(

dataset_props = DatasetPropertiesClass(
name=report_info.get("name") if is_mode_dataset else query_data.get("name"),
description=f"""### Source Code
``` sql
{query_data.get("raw_query")}
```
""",
description=None,
externalUrl=externalUrl,
customProperties=self.get_custom_props_from_dict(
query_data,
[
"id" "created_at",
"id",
"created_at",
"updated_at",
"last_run_id",
"data_source_id",
Expand All @@ -949,14 +947,23 @@ def construct_query_or_dataset(
],
),
)

yield (
MetadataChangeProposalWrapper(
entityUrn=query_urn,
aspect=dataset_props,
).as_workunit()
)

if raw_query := query_data.get("raw_query"):
yield MetadataChangeProposalWrapper(
entityUrn=query_urn,
aspect=ViewPropertiesClass(
viewLogic=raw_query,
viewLanguage=QueryLanguageClass.SQL,
materialized=False,
),
).as_workunit()

if is_mode_dataset:
space_container_key = self.gen_space_key(space_token)
yield from add_dataset_to_container(
Expand Down
43 changes: 41 additions & 2 deletions metadata-ingestion/tests/integration/mode/mode_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@
"datasets": [
"urn:li:dataset:(urn:li:dataPlatform:mode,5450544,PROD)"
],
"dashboards": [],
"lastModified": {
"created": {
"time": 1639169724316,
Expand Down Expand Up @@ -253,14 +254,15 @@
"aspect": {
"json": {
"customProperties": {
"id": "19780522",
"created_at": "2024-09-02T07:38:43.755Z",
"updated_at": "2024-09-02T07:40:44.046Z",
"last_run_id": "3535709679",
"data_source_id": "44763",
"report_imports_count": "2"
},
"externalUrl": "https://app.mode.com/acryl/datasets/24f66e1701b6",
"name": "Dataset 1",
"description": "### Source Code\n``` sql\n-- Returns first 100 rows from DATAHUB_COMMUNITY.POSTGRES_PUBLIC.COMPANY\n SELECT \n\t\tAGE,\n\t\tID,\n\t\tNAME,\n\t\t_FIVETRAN_DELETED,\n\t\t_FIVETRAN_SYNCED\n FROM DATAHUB_COMMUNITY.POSTGRES_PUBLIC.COMPANY LIMIT 100;\n\n-- Returns first 100 rows from ETHAN_TEST_DB.PUBLIC.ACCOUNT_PHONE_NUMBER\n SELECT \n\t\tCOMMUNICATION_ACCOUNT_ID,\n\t\tID,\n\t\tMMS_CAPABLE,\n\t\tPHONE_NUMBER,\n\t\tSMS_CAPABLE,\n\t\tSTATUS,\n\t\tSTATUS_TLM,\n\t\tTLM,\n\t\tVOICE_CAPABLE,\n\t\tWHEN_CREATED\n FROM ETHAN_TEST_DB.PUBLIC.ACCOUNT_PHONE_NUMBER LIMIT 100;\n \n \n```\n ",
"tags": []
}
},
Expand All @@ -270,6 +272,24 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mode,5450544,PROD)",
"changeType": "UPSERT",
"aspectName": "viewProperties",
"aspect": {
"json": {
"materialized": false,
"viewLogic": "-- Returns first 100 rows from DATAHUB_COMMUNITY.POSTGRES_PUBLIC.COMPANY\n SELECT \n\t\tAGE,\n\t\tID,\n\t\tNAME,\n\t\t_FIVETRAN_DELETED,\n\t\t_FIVETRAN_SYNCED\n FROM DATAHUB_COMMUNITY.POSTGRES_PUBLIC.COMPANY LIMIT 100;\n\n-- Returns first 100 rows from ETHAN_TEST_DB.PUBLIC.ACCOUNT_PHONE_NUMBER\n SELECT \n\t\tCOMMUNICATION_ACCOUNT_ID,\n\t\tID,\n\t\tMMS_CAPABLE,\n\t\tPHONE_NUMBER,\n\t\tSMS_CAPABLE,\n\t\tSTATUS,\n\t\tSTATUS_TLM,\n\t\tTLM,\n\t\tVOICE_CAPABLE,\n\t\tWHEN_CREATED\n FROM ETHAN_TEST_DB.PUBLIC.ACCOUNT_PHONE_NUMBER LIMIT 100;\n \n ",
"viewLanguage": "SQL"
}
},
"systemMetadata": {
"lastObserved": 1638860400000,
"runId": "mode-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mode,5450544,PROD)",
Expand Down Expand Up @@ -336,13 +356,14 @@
"aspect": {
"json": {
"customProperties": {
"id": "10149707",
"created_at": "2021-12-10T20:55:24.361Z",
"updated_at": "2021-12-10T23:12:53.273Z",
"last_run_id": "1897576958",
"data_source_id": "34499"
},
"externalUrl": "https://app.mode.com/acryl/reports/9d2da37fa91e/details/queries/6e26a9f3d4e2",
"name": "Customer and staff",
"description": "### Source Code\n``` sql\nSELECT rental.*, staff.first_name \"Staff First Name\", staff.last_name \"Staff Last Name\" FROM {{ @join_on_definition as rental }} join staff on staff.staff_id = rental.staff_id where selected_id = {{ selected_id }} \n{% form %}\nselected_id:\n type: text\n default: my_id\n{% endform %}\n```\n ",
"tags": []
}
},
Expand All @@ -352,6 +373,24 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mode,10149707,PROD)",
"changeType": "UPSERT",
"aspectName": "viewProperties",
"aspect": {
"json": {
"materialized": false,
"viewLogic": "SELECT rental.*, staff.first_name \"Staff First Name\", staff.last_name \"Staff Last Name\" FROM {{ @join_on_definition as rental }} join staff on staff.staff_id = rental.staff_id where selected_id = {{ selected_id }} \n{% form %}\nselected_id:\n type: text\n default: my_id\n{% endform %}",
"viewLanguage": "SQL"
}
},
"systemMetadata": {
"lastObserved": 1638860400000,
"runId": "mode-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mode,10149707,PROD)",
Expand Down

0 comments on commit 5ec0ec9

Please sign in to comment.