From b6c7fe8267dd55dbe87c06ed07a8e7ff4bfcc057 Mon Sep 17 00:00:00 2001 From: Shubham Jagtap <132359390+shubhamjagtap639@users.noreply.github.com> Date: Tue, 9 Jul 2024 09:45:20 +0530 Subject: [PATCH] refactor(ingestion): remove company domain for security reason (#10839) --- .../powerbi/golden_test_admin_only.json | 4 +- .../integration/powerbi/golden_test_cll.json | 2 +- .../powerbi/golden_test_container.json | 4 +- .../golden_test_disabled_ownership.json | 2 +- .../powerbi/golden_test_endorsement.json | 2 +- .../powerbi/golden_test_ingest.json | 2 +- .../powerbi/golden_test_lineage.json | 2 +- .../golden_test_lower_case_urn_ingest.json | 2 +- .../golden_test_platform_instance_ingest.json | 2 +- .../powerbi/golden_test_report.json | 4 +- .../golden_test_scan_all_workspaces.json | 2 +- ...lden_test_server_to_platform_instance.json | 2 +- .../powerbi/test_admin_only_api.py | 2 +- .../integration/powerbi/test_m_parser.py | 2 +- .../tests/integration/powerbi/test_powerbi.py | 2 +- .../golden_test_platform_instance_ingest.json | 25 +++++--- .../golden_test_qlik_sense_ingest.json | 25 +++++--- .../integration/qlik_sense/test_qlik_sense.py | 4 +- .../tests/integration/sigma/test_sigma.py | 8 +-- .../tests/unit/test_transform_dataset.py | 64 +++++++++---------- 20 files changed, 86 insertions(+), 76 deletions(-) diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json index 4ca5cc4afd00d8..29e9ccebf067ec 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json @@ -390,7 +390,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, @@ -1709,7 +1709,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json index 4a08c6658c3950..937cad0b9ec17e 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json @@ -483,7 +483,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json index b65ba71c80159a..501ec284097b39 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json @@ -780,7 +780,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, @@ -2213,7 +2213,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json index 43a5aad3cd9674..74779ac7a0577e 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json @@ -382,7 +382,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json index cf448776ba4149..442dfd5c8c0825 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json @@ -482,7 +482,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json index 566e405a03abbc..2c4ff6ee851f4a 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json @@ -382,7 +382,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json index 4f530acb11c80f..0aea8514559ecb 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json @@ -465,7 +465,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json index 37ba034b08fe26..22a00236af8c61 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json @@ -382,7 +382,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json index 12ce114482eeeb..cf5d4df460e231 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json @@ -382,7 +382,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json index 43092c4e20acbe..cce9e3f8755ba5 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json @@ -382,7 +382,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, @@ -1493,7 +1493,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json index 3c4499765e525b..5e244e0e0f93f7 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json @@ -382,7 +382,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json index 0c6239ae8d3319..8fd8989b81122b 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json @@ -490,7 +490,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.GSLAB.COM\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", "viewLanguage": "m_query" } }, diff --git a/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py b/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py index bb572fe3a7ae5a..91c60825243892 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py +++ b/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py @@ -283,7 +283,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None "name": "job-history", "source": [ { - "expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.GSLAB.COM", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1', + "expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1', } ], "datasourceUsages": [ diff --git a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py index 0aa290cecd8fd4..e98b64c011fd83 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py +++ b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py @@ -35,7 +35,7 @@ 'let\n Source = Sql.Database("AUPRDWHDB", "COMMOPSDB", [Query="Select#(lf)*,#(lf)concat((UPPER(REPLACE(SALES_SPECIALIST,\'-\',\'\'))),#(lf)LEFT(CAST(INVOICE_DATE AS DATE),4)+LEFT(RIGHT(CAST(INVOICE_DATE AS DATE),5),2)) AS AGENT_KEY,#(lf)CASE#(lf) WHEN CLASS = \'Software\' and (NOT(PRODUCT in (\'ADV\', \'Adv\') and left(ACCOUNT_ID,2)=\'10\') #(lf) or V_ENTERPRISE_INVOICED_REVENUE.TYPE = \'Manual Adjustment\') THEN INVOICE_AMOUNT#(lf) WHEN V_ENTERPRISE_INVOICED_REVENUE.TYPE IN (\'Recurring\',\'0\') THEN INVOICE_AMOUNT#(lf) ELSE 0#(lf)END as SOFTWARE_INV#(lf)#(lf)from V_ENTERPRISE_INVOICED_REVENUE", CommandTimeout=#duration(0, 1, 30, 0)]),\n #"Added Conditional Column" = Table.AddColumn(Source, "Services", each if [CLASS] = "Services" then [INVOICE_AMOUNT] else 0),\n #"Added Custom" = Table.AddColumn(#"Added Conditional Column", "Advanced New Sites", each if [PRODUCT] = "ADV"\nor [PRODUCT] = "Adv"\nthen [NEW_SITE]\nelse 0)\nin\n #"Added Custom"', 'let\n Source = Snowflake.Databases("ghh48144.snowflakefakecomputing.com","GSL_TEST_WH",[Role="ACCOUNTADMIN"]),\n Source2 = PostgreSQL.Database("localhost", "mics"),\n public_order_date = Source2{[Schema="public",Item="order_date"]}[Data],\n GSL_TEST_DB_Database = Source{[Name="GSL_TEST_DB",Kind="Database"]}[Data],\n PUBLIC_Schema = GSL_TEST_DB_Database{[Name="PUBLIC",Kind="Schema"]}[Data],\n SALES_ANALYST_VIEW_View = PUBLIC_Schema{[Name="SALES_ANALYST_VIEW",Kind="View"]}[Data],\n two_source_table = Table.Combine({public_order_date, SALES_ANALYST_VIEW_View})\n in\n two_source_table', 'let\n Source = PostgreSQL.Database("localhost" , "mics" ),\n public_order_date = Source{[Schema="public",Item="order_date"]}[Data] \n in \n public_order_date', - 'let\n Source = Oracle.Database("localhost:1521/salesdb.GSLAB.COM", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1', + 'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1', 'let\n Source = Sql.Database("localhost", "library"),\n dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data]\n in dbo_book_issue', 'let\n Source = Snowflake.Databases("ghh48144.snowflakefakecomputing.com","GSL_TEST_WH",[Role="ACCOUNTADMIN"]),\n GSL_TEST_DB_Database = Source{[Name="GSL_TEST_DB",Kind="Database"]}[Data],\n PUBLIC_Schema = GSL_TEST_DB_Database{[Name="PUBLIC",Kind="Schema"]}[Data],\n SALES_FORECAST_Table = PUBLIC_Schema{[Name="SALES_FORECAST",Kind="Table"]}[Data],\n SALES_ANALYST_Table = PUBLIC_Schema{[Name="SALES_ANALYST",Kind="Table"]}[Data],\n RESULT = Table.Combine({SALES_FORECAST_Table, SALES_ANALYST_Table})\n\nin\n RESULT', 'let\n Source = GoogleBigQuery.Database(),\n #"seraphic-music-344307" = Source{[Name="seraphic-music-344307"]}[Data],\n school_dataset_Schema = #"seraphic-music-344307"{[Name="school_dataset",Kind="Schema"]}[Data],\n first_Table = school_dataset_Schema{[Name="first",Kind="Table"]}[Data]\nin\n first_Table', diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index fafa02d5301e0f..06cc40fe7b24c1 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -393,7 +393,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "name": "job-history", "source": [ { - "expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.GSLAB.COM", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1', + "expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1', } ], "datasourceUsages": [ diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json index 6917b833d05a35..089c80f5c4ee73 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_platform_instance_ingest.json @@ -89,10 +89,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -100,7 +101,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327546106, + "lastObserved": 1720073671135, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -319,10 +320,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -330,7 +332,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327546129, + "lastObserved": 1720073671192, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -469,10 +471,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -480,7 +483,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327546135, + "lastObserved": 1720073671209, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1217,10 +1220,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1228,7 +1232,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327546161, + "lastObserved": 1720073671300, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1414,10 +1418,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1425,7 +1430,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327546168, + "lastObserved": 1720073671321, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json index 5b47c09a791430..f285fc17a6fb47 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json +++ b/metadata-ingestion/tests/integration/qlik_sense/golden_test_qlik_sense_ingest.json @@ -87,10 +87,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -98,7 +99,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327442987, + "lastObserved": 1720073670616, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -303,10 +304,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -314,7 +316,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327443016, + "lastObserved": 1720073670661, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -432,10 +434,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -443,7 +446,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327443023, + "lastObserved": 1720073670669, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1113,10 +1116,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1124,7 +1128,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327443058, + "lastObserved": 1720073670731, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } @@ -1289,10 +1293,11 @@ "json": { "owners": [ { - "owner": "urn:li:corpuser:Shubham jagtap", + "owner": "urn:li:corpuser:john doe", "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -1300,7 +1305,7 @@ } }, "systemMetadata": { - "lastObserved": 1707327443073, + "lastObserved": 1720073670744, "runId": "qlik-sense-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py index 818bc5198a5b14..ee1aafb6cf32dc 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py +++ b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py @@ -204,8 +204,8 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "tenantId": "ysA4KqhDrbdy36hO9wwo4HUvPxeaKT7A", "status": "active", "subject": "auth0|fd95ee6facf82e692d2eac4ccb5ddb18ef05c22a7575fcc4d26d7bc9aefedb4f", - "name": "Shubham jagtap", - "email": "Shubham.Jagtap@gslab.com", + "name": "john doe", + "email": "john.doe@example.com", "roles": [ "TenantAdmin", "AnalyticsAdmin", diff --git a/metadata-ingestion/tests/integration/sigma/test_sigma.py b/metadata-ingestion/tests/integration/sigma/test_sigma.py index f4948de7ea8250..b6e9db99eed39a 100644 --- a/metadata-ingestion/tests/integration/sigma/test_sigma.py +++ b/metadata-ingestion/tests/integration/sigma/test_sigma.py @@ -264,7 +264,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "json": { "elementId": "kH0MeihtGs", "name": "ADOPTIONS", - "sql": 'select PK "Pk", PET_FK "Pet Fk", HUMAN_FK "Human Fk", STATUS "Status", CAST_TIMESTAMP_TO_DATETIME_7 "Created At", CAST_TIMESTAMP_TO_DATETIME_8 "Updated At" from (select PK, PET_FK, HUMAN_FK, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_8 from (select * from LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS ADOPTIONS limit 1000) Q1) Q2 limit 1000\n\n-- Sigma Σ {"request-id":"3d4bf15e-6a17-4967-ad2a-213341233a04","email":"Shubham.Jagtap@gslab.com"}', + "sql": 'select PK "Pk", PET_FK "Pet Fk", HUMAN_FK "Human Fk", STATUS "Status", CAST_TIMESTAMP_TO_DATETIME_7 "Created At", CAST_TIMESTAMP_TO_DATETIME_8 "Updated At" from (select PK, PET_FK, HUMAN_FK, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_8 from (select * from LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS ADOPTIONS limit 1000) Q1) Q2 limit 1000\n\n-- Sigma Σ {"request-id":"3d4bf15e-6a17-4967-ad2a-213341233a04","email":"john.doe@example.com"}', }, }, "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/lineage/elements/Ml9C5ezT5W": { @@ -299,7 +299,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "json": { "elementId": "Ml9C5ezT5W", "name": "Count of Profile Id by Status", - "sql": 'with Q1 as (select PK, PROFILE_ID, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_6, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) select STATUS_10 "Status", COUNT_23 "Count of Profile Id", PK_8 "Pk", PROFILE_ID_9 "Profile Id", CAST_TIMESTAMP_TO_DATETIME_8 "Created At", CAST_TIMESTAMP_TO_DATETIME_9 "Updated At" from (select Q3.PK_8 PK_8, Q3.PROFILE_ID_9 PROFILE_ID_9, Q3.STATUS_10 STATUS_10, Q3.CAST_TIMESTAMP_TO_DATETIME_8 CAST_TIMESTAMP_TO_DATETIME_8, Q3.CAST_TIMESTAMP_TO_DATETIME_9 CAST_TIMESTAMP_TO_DATETIME_9, Q6.COUNT_23 COUNT_23, Q6.STATUS_11 STATUS_11 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CAST_TIMESTAMP_TO_DATETIME_6 CAST_TIMESTAMP_TO_DATETIME_8, CAST_TIMESTAMP_TO_DATETIME_7 CAST_TIMESTAMP_TO_DATETIME_9 from Q1 Q2 order by STATUS_10 asc limit 1000) Q3 left join (select count(PROFILE_ID_9) COUNT_23, STATUS_10 STATUS_11 from (select PROFILE_ID PROFILE_ID_9, STATUS STATUS_10 from Q1 Q4) Q5 group by STATUS_10) Q6 on equal_null(Q3.STATUS_10, Q6.STATUS_11)) Q8 order by STATUS_10 asc limit 1000\n\n-- Sigma Σ {"request-id":"988dd6b5-0678-4421-ae14-21594c0ee97a","email":"Shubham.Jagtap@gslab.com"}', + "sql": 'with Q1 as (select PK, PROFILE_ID, STATUS, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_6, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_7 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) select STATUS_10 "Status", COUNT_23 "Count of Profile Id", PK_8 "Pk", PROFILE_ID_9 "Profile Id", CAST_TIMESTAMP_TO_DATETIME_8 "Created At", CAST_TIMESTAMP_TO_DATETIME_9 "Updated At" from (select Q3.PK_8 PK_8, Q3.PROFILE_ID_9 PROFILE_ID_9, Q3.STATUS_10 STATUS_10, Q3.CAST_TIMESTAMP_TO_DATETIME_8 CAST_TIMESTAMP_TO_DATETIME_8, Q3.CAST_TIMESTAMP_TO_DATETIME_9 CAST_TIMESTAMP_TO_DATETIME_9, Q6.COUNT_23 COUNT_23, Q6.STATUS_11 STATUS_11 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CAST_TIMESTAMP_TO_DATETIME_6 CAST_TIMESTAMP_TO_DATETIME_8, CAST_TIMESTAMP_TO_DATETIME_7 CAST_TIMESTAMP_TO_DATETIME_9 from Q1 Q2 order by STATUS_10 asc limit 1000) Q3 left join (select count(PROFILE_ID_9) COUNT_23, STATUS_10 STATUS_11 from (select PROFILE_ID PROFILE_ID_9, STATUS STATUS_10 from Q1 Q4) Q5 group by STATUS_10) Q6 on equal_null(Q3.STATUS_10, Q6.STATUS_11)) Q8 order by STATUS_10 asc limit 1000\n\n-- Sigma Σ {"request-id":"988dd6b5-0678-4421-ae14-21594c0ee97a","email":"john.doe@example.com"}', }, }, "https://aws-api.sigmacomputing.com/v2/workbooks/9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b/pages/DFSieiAcgo/elements": { @@ -375,7 +375,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "json": { "elementId": "tQJu5N1l81", "name": "PETS ADOPTIONS JOIN", - "sql": 'select PK_8 "Pk", PROFILE_ID_9 "Profile Id", STATUS_10 "Status", CAST_TIMESTAMP_TO_DATETIME_11 "Created At", CAST_TIMESTAMP_TO_DATETIME_12 "Updated At", PK_13 "Pk (ADOPTIONS)", PET_FK_14 "Pet Fk", HUMAN_FK_15 "Human Fk", STATUS_16 "Status (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_19 "Created At (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_20 "Updated At (ADOPTIONS)" from (select PK_8, PROFILE_ID_9, STATUS_10, CAST_TIMESTAMP_TO_DATETIME_11, CAST_TIMESTAMP_TO_DATETIME_12, PK_13, PET_FK_14, HUMAN_FK_15, STATUS_16, CREATED_AT_17::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_19, UPDATED_AT_18::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_20 from (select Q1.PK_8 PK_8, Q1.PROFILE_ID_9 PROFILE_ID_9, Q1.STATUS_10 STATUS_10, Q1.CAST_TIMESTAMP_TO_DATETIME_11 CAST_TIMESTAMP_TO_DATETIME_11, Q1.CAST_TIMESTAMP_TO_DATETIME_12 CAST_TIMESTAMP_TO_DATETIME_12, Q2.PK PK_13, Q2.PET_FK PET_FK_14, Q2.HUMAN_FK HUMAN_FK_15, Q2.STATUS STATUS_16, Q2.CREATED_AT CREATED_AT_17, Q2.UPDATED_AT UPDATED_AT_18 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_11, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_12 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) Q1 inner join LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS Q2 on (Q1.PK_8 = Q2.PET_FK) limit 1000) Q4) Q5 limit 1000\n\n-- Sigma Σ {"request-id":"f5a997ef-b80c-47f1-b32e-9cd0f50cd491","email":"Shubham.Jagtap@gslab.com"}', + "sql": 'select PK_8 "Pk", PROFILE_ID_9 "Profile Id", STATUS_10 "Status", CAST_TIMESTAMP_TO_DATETIME_11 "Created At", CAST_TIMESTAMP_TO_DATETIME_12 "Updated At", PK_13 "Pk (ADOPTIONS)", PET_FK_14 "Pet Fk", HUMAN_FK_15 "Human Fk", STATUS_16 "Status (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_19 "Created At (ADOPTIONS)", CAST_TIMESTAMP_TO_DATETIME_20 "Updated At (ADOPTIONS)" from (select PK_8, PROFILE_ID_9, STATUS_10, CAST_TIMESTAMP_TO_DATETIME_11, CAST_TIMESTAMP_TO_DATETIME_12, PK_13, PET_FK_14, HUMAN_FK_15, STATUS_16, CREATED_AT_17::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_19, UPDATED_AT_18::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_20 from (select Q1.PK_8 PK_8, Q1.PROFILE_ID_9 PROFILE_ID_9, Q1.STATUS_10 STATUS_10, Q1.CAST_TIMESTAMP_TO_DATETIME_11 CAST_TIMESTAMP_TO_DATETIME_11, Q1.CAST_TIMESTAMP_TO_DATETIME_12 CAST_TIMESTAMP_TO_DATETIME_12, Q2.PK PK_13, Q2.PET_FK PET_FK_14, Q2.HUMAN_FK HUMAN_FK_15, Q2.STATUS STATUS_16, Q2.CREATED_AT CREATED_AT_17, Q2.UPDATED_AT UPDATED_AT_18 from (select PK PK_8, PROFILE_ID PROFILE_ID_9, STATUS STATUS_10, CREATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_11, UPDATED_AT::timestamp_ltz CAST_TIMESTAMP_TO_DATETIME_12 from LONG_TAIL_COMPANIONS.ADOPTION.PETS PETS) Q1 inner join LONG_TAIL_COMPANIONS.ADOPTION.ADOPTIONS Q2 on (Q1.PK_8 = Q2.PET_FK) limit 1000) Q4) Q5 limit 1000\n\n-- Sigma Σ {"request-id":"f5a997ef-b80c-47f1-b32e-9cd0f50cd491","email":"john.doe@example.com"}', }, }, "https://aws-api.sigmacomputing.com/v2/members": { @@ -388,7 +388,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "memberType": "admin", "firstName": "Shubham", "lastName": "Jagtap", - "email": "Shubham.Jagtap@gslab.com", + "email": "john.doe@example.com", "profileImgUrl": None, "createdBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", "updatedBy": "CPbEdA26GNQ2cM2Ra2BeO0fa5Awz1", diff --git a/metadata-ingestion/tests/unit/test_transform_dataset.py b/metadata-ingestion/tests/unit/test_transform_dataset.py index 4170fb5bf8b678..4c4e1620f4faae 100644 --- a/metadata-ingestion/tests/unit/test_transform_dataset.py +++ b/metadata-ingestion/tests/unit/test_transform_dataset.py @@ -1941,7 +1941,7 @@ def test_simple_add_dataset_domain_aspect_name(mock_datahub_graph): def test_simple_add_dataset_domain(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" @@ -1950,7 +1950,7 @@ def test_simple_add_dataset_domain(mock_datahub_graph): output = run_dataset_transformer_pipeline( transformer_type=SimpleAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={"domains": [acryl_domain]}, pipeline_context=pipeline_context, ) @@ -1963,13 +1963,13 @@ def test_simple_add_dataset_domain(mock_datahub_graph): assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 2 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains def test_simple_add_dataset_domain_mce_support(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" @@ -1979,7 +1979,7 @@ def test_simple_add_dataset_domain_mce_support(mock_datahub_graph): output = run_dataset_transformer_pipeline( transformer_type=SimpleAddDatasetDomain, aspect=None, - config={"domains": [gslab_domain, acryl_domain]}, + config={"domains": [datahub_domain, acryl_domain]}, pipeline_context=pipeline_context, use_mce=True, ) @@ -1994,13 +1994,13 @@ def test_simple_add_dataset_domain_mce_support(mock_datahub_graph): assert isinstance(output[1].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[1].record.aspect) assert len(transformed_aspect.domains) == 2 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pipeline_context: PipelineContext = PipelineContext( run_id="test_simple_add_dataset_domain" @@ -2009,7 +2009,7 @@ def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph): output = run_dataset_transformer_pipeline( transformer_type=SimpleAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={"replace_existing": True, "domains": [acryl_domain]}, pipeline_context=pipeline_context, ) @@ -2022,13 +2022,13 @@ def test_simple_add_dataset_domain_replace_existing(mock_datahub_graph): assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 1 - assert gslab_domain not in transformed_aspect.domains + assert datahub_domain not in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains def test_simple_add_dataset_domain_semantics_overwrite(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") server_domain = builder.make_domain_urn("test.io") pipeline_context = PipelineContext(run_id="transformer_pipe_line") @@ -2042,7 +2042,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: output = run_dataset_transformer_pipeline( transformer_type=SimpleAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "semantics": TransformerSemantics.OVERWRITE, "domains": [acryl_domain], @@ -2058,7 +2058,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 2 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains assert server_domain not in transformed_aspect.domains @@ -2067,7 +2067,7 @@ def test_simple_add_dataset_domain_semantics_patch( pytestconfig, tmp_path, mock_time, mock_datahub_graph ): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") server_domain = builder.make_domain_urn("test.io") pipeline_context = PipelineContext(run_id="transformer_pipe_line") @@ -2081,7 +2081,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: output = run_dataset_transformer_pipeline( transformer_type=SimpleAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "replace_existing": False, "semantics": TransformerSemantics.PATCH, @@ -2098,7 +2098,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 3 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains assert server_domain in transformed_aspect.domains @@ -2117,7 +2117,7 @@ def test_pattern_add_dataset_domain_aspect_name(mock_datahub_graph): def test_pattern_add_dataset_domain_match(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*" pipeline_context: PipelineContext = PipelineContext( @@ -2127,7 +2127,7 @@ def test_pattern_add_dataset_domain_match(mock_datahub_graph): output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "domain_pattern": {"rules": {pattern: [acryl_domain]}}, }, @@ -2142,13 +2142,13 @@ def test_pattern_add_dataset_domain_match(mock_datahub_graph): assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 2 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains def test_pattern_add_dataset_domain_no_match(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:invalid,.*" pipeline_context: PipelineContext = PipelineContext( @@ -2158,7 +2158,7 @@ def test_pattern_add_dataset_domain_no_match(mock_datahub_graph): output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "domain_pattern": {"rules": {pattern: [acryl_domain]}}, }, @@ -2173,13 +2173,13 @@ def test_pattern_add_dataset_domain_no_match(mock_datahub_graph): assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 1 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain not in transformed_aspect.domains def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*" pipeline_context: PipelineContext = PipelineContext( @@ -2189,7 +2189,7 @@ def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph): output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "replace_existing": True, "domain_pattern": {"rules": {pattern: [acryl_domain]}}, @@ -2205,13 +2205,13 @@ def test_pattern_add_dataset_domain_replace_existing_match(mock_datahub_graph): assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 1 - assert gslab_domain not in transformed_aspect.domains + assert datahub_domain not in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:invalid,.*" pipeline_context: PipelineContext = PipelineContext( @@ -2221,7 +2221,7 @@ def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "replace_existing": True, "domain_pattern": {"rules": {pattern: [acryl_domain]}}, @@ -2241,7 +2241,7 @@ def test_pattern_add_dataset_domain_replace_existing_no_match(mock_datahub_graph def test_pattern_add_dataset_domain_semantics_overwrite(mock_datahub_graph): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") server_domain = builder.make_domain_urn("test.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*" @@ -2256,7 +2256,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "semantics": TransformerSemantics.OVERWRITE, "domain_pattern": {"rules": {pattern: [acryl_domain]}}, @@ -2272,7 +2272,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 2 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains assert server_domain not in transformed_aspect.domains @@ -2281,7 +2281,7 @@ def test_pattern_add_dataset_domain_semantics_patch( pytestconfig, tmp_path, mock_time, mock_datahub_graph ): acryl_domain = builder.make_domain_urn("acryl.io") - gslab_domain = builder.make_domain_urn("gslab.io") + datahub_domain = builder.make_domain_urn("datahubproject.io") server_domain = builder.make_domain_urn("test.io") pattern = "urn:li:dataset:\\(urn:li:dataPlatform:bigquery,.*" @@ -2296,7 +2296,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: output = run_dataset_transformer_pipeline( transformer_type=PatternAddDatasetDomain, - aspect=models.DomainsClass(domains=[gslab_domain]), + aspect=models.DomainsClass(domains=[datahub_domain]), config={ "replace_existing": False, "semantics": TransformerSemantics.PATCH, @@ -2313,7 +2313,7 @@ def fake_get_domain(entity_urn: str) -> models.DomainsClass: assert isinstance(output[0].record.aspect, models.DomainsClass) transformed_aspect = cast(models.DomainsClass, output[0].record.aspect) assert len(transformed_aspect.domains) == 3 - assert gslab_domain in transformed_aspect.domains + assert datahub_domain in transformed_aspect.domains assert acryl_domain in transformed_aspect.domains assert server_domain in transformed_aspect.domains