diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py index be9a3b20aaba1..46694dfcc47d1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau.py @@ -749,11 +749,15 @@ def get_connection_object_page( count: int = 0, offset: int = 0, retry_on_auth_error: bool = True, + retries_remaining: Optional[int] = None, ) -> Tuple[dict, int, int]: + retries_remaining = retries_remaining or self.config.max_retries + logger.debug( f"Query {connection_type} to get {count} objects with offset {offset}" ) try: + assert self.server is not None query_data = query_metadata( self.server, query, connection_type, count, offset, query_filter ) @@ -766,7 +770,31 @@ def get_connection_object_page( # will be thrown and we need to re-authenticate and retry. self._authenticate() return self.get_connection_object_page( - query, connection_type, query_filter, count, offset, False + query, + connection_type, + query_filter, + count, + offset, + retry_on_auth_error=False, + retries_remaining=retries_remaining, + ) + except OSError: + # In tableauseverclient 0.26 (which was yanked and released in 0.28 on 2023-10-04), + # the request logic was changed to use threads. + # https://github.com/tableau/server-client-python/commit/307d8a20a30f32c1ce615cca7c6a78b9b9bff081 + # I'm not exactly sure why, but since then, we now occasionally see + # `OSError: Response is not a http response?` for some requests. This + # retry logic is basically a bandaid for that. + if retries_remaining <= 0: + raise + return self.get_connection_object_page( + query, + connection_type, + query_filter, + count, + offset, + retry_on_auth_error=False, + retries_remaining=retries_remaining - 1, ) if c.ERRORS in query_data: @@ -781,11 +809,7 @@ def get_connection_object_page( else: raise RuntimeError(f"Query {connection_type} error: {errors}") - connection_object = ( - query_data.get(c.DATA).get(connection_type, {}) - if query_data.get(c.DATA) - else {} - ) + connection_object = query_data.get(c.DATA, {}).get(connection_type, {}) total_count = connection_object.get(c.TOTAL_COUNT, 0) has_next_page = connection_object.get(c.PAGE_INFO, {}).get( @@ -2520,3 +2544,4 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: def get_report(self) -> TableauSourceReport: return self.report + return self.report diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py index 65d779b7f4516..a2f460feca388 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py @@ -5,6 +5,7 @@ from typing import Dict, List, Optional, Tuple from pydantic.fields import Field +from tableauserverclient import Server import datahub.emitter.mce_builder as builder from datahub.configuration.common import ConfigModel @@ -699,7 +700,6 @@ def get_overridden_info( platform_instance_map: Optional[Dict[str, str]], lineage_overrides: Optional[TableauLineageOverrides] = None, ) -> Tuple[Optional[str], Optional[str], str, str]: - original_platform = platform = get_platform(connection_type) if ( lineage_overrides is not None @@ -824,7 +824,14 @@ def clean_query(query: str) -> str: return query -def query_metadata(server, main_query, connection_name, first, offset, qry_filter=""): +def query_metadata( + server: Server, + main_query: str, + connection_name: str, + first: int, + offset: int, + qry_filter: str = "", +) -> dict: query = """{{ {connection_name} (first:{first}, offset:{offset}, filter:{{{filter}}}) {{ diff --git a/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js b/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js index a2d4de0f51659..59af6daf9b8f6 100644 --- a/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js +++ b/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js @@ -1,53 +1,94 @@ const domainName = "CypressNestedDomain"; const domainDescription = "CypressNestedDomainDescription"; -describe("nested domains test", () => { +const createDomain = () => { + cy.clickOptionWithTestId("domains-new-domain-button"); + cy.get('[data-testid="create-domain-name"]').click().type(domainName); + cy.get('[data-testid="create-domain-description"]').click().type(domainDescription); + cy.clickOptionWithTestId("create-domain-button"); + cy.waitTextVisible("Created domain!"); + } - it("create a domain, move under parent, remove domain", () => { - // Create a new domain without a parent - cy.loginWithCredentials(); - cy.goToDomainList(); - cy.clickOptionWithTestId("domains-new-domain-button"); - cy.get('[data-testid="create-domain-name"]').click().type(domainName); - cy.get('[data-testid="create-domain-description"]').click().type(domainDescription); - cy.clickOptionWithTestId("create-domain-button"); - cy.waitTextVisible(domainName); +const moveDomaintoRootLevel = () => { + cy.clickOptionWithText(domainName); + cy.openThreeDotDropdown(); + cy.clickOptionWithTestId("entity-menu-move-button"); + cy.get('[data-testid="move-domain-modal"]').contains("Marketing").click({force: true}); + cy.waitTextVisible('Marketing') + cy.clickOptionWithTestId("move-domain-modal-move-button") + } - // Ensure the new domain has no parent in the navigation sidebar - cy.waitTextVisible(domainDescription); +const moveDomaintoParent = () => { + cy.get('[data-testid="domain-list-item"]').contains("Marketing").prev().click(); + cy.clickOptionWithText(domainName); + cy.waitTextVisible(domainName) + cy.openThreeDotDropdown(); + cy.clickOptionWithTestId("entity-menu-move-button"); + cy.clickOptionWithTestId("move-domain-modal-move-button") + } - // Move a domain from the root level to be under a parent domain - cy.clickOptionWithText(domainName); - cy.openThreeDotDropdown(); - cy.clickOptionWithTestId("entity-menu-move-button"); - cy.get('[data-testid="move-domain-modal"]').contains("Marketing").click({force: true}); - cy.get('[data-testid="move-domain-modal"]').contains("Marketing").should("be.visible"); - cy.clickOptionWithTestId("move-domain-modal-move-button").wait(5000); + const deleteFromDomainDropdown = () => { + cy.clickOptionWithText('Filters') + cy.openThreeDotDropdown(); + cy.clickOptionWithTestId("entity-menu-delete-button"); + cy.waitTextVisible("Are you sure you want to remove this Domain?"); + cy.clickOptionWithText("Yes"); + } - // Wnsure domain is no longer on the sidebar navigator at the top level but shows up under the parent +const deleteDomain = () => { + cy.clickOptionWithText(domainName).waitTextVisible('Domains'); + deleteFromDomainDropdown() + } + +//Delete Unecessary Existing Domains + const deleteExisitingDomain = () => { + cy.get('a[href*="urn:li"] span[class^="ant-typography"]') + .should('be.visible') + .its('length') + .then((length) => { + for (let i = 0; i < length - 1; i++) { + cy.get('a[href*="urn:li"] span[class^="ant-typography"]') + .should('be.visible') + .first() + .click({ force: true }); + deleteFromDomainDropdown(); + } + }); + cy.waitTextVisible('My custom domain'); + } + +describe("nested domains test", () => { + beforeEach (() => { + cy.loginWithCredentials(); + cy.goToDomainList(); + }); + + it("Create a new domain", () => { + deleteExisitingDomain() + createDomain(); + cy.waitTextVisible("Domains"); + }); + + it("Move domain root level to parent level", () => { + cy.waitTextVisible(domainName) + moveDomaintoRootLevel(); + cy.waitTextVisible("Moved Domain!") cy.goToDomainList(); - cy.ensureTextNotPresent(domainName); - cy.ensureTextNotPresent(domainDescription); cy.waitTextVisible("1 sub-domain"); + }); - // Move a domain from under a parent domain to the root level - cy.get('[data-testid="domain-list-item"]').contains("Marketing").prev().click(); - cy.clickOptionWithText(domainName); - cy.openThreeDotDropdown(); - cy.clickOptionWithTestId("entity-menu-move-button"); - cy.clickOptionWithTestId("move-domain-modal-move-button").wait(5000); + it("Move domain parent level to root level", () => { + moveDomaintoParent(); + cy.waitTextVisible("Moved Domain!") cy.goToDomainList(); cy.waitTextVisible(domainName); cy.waitTextVisible(domainDescription); + }); - // Delete a domain - cy.clickOptionWithText(domainName).wait(3000); - cy.openThreeDotDropdown(); - cy.clickOptionWithTestId("entity-menu-delete-button"); - cy.waitTextVisible("Are you sure you want to remove this Domain?"); - cy.clickOptionWithText("Yes"); - cy.waitTextVisible("Deleted Domain!"); + it("Remove the domain", () => { + deleteDomain(); + cy.goToDomainList(); cy.ensureTextNotPresent(domainName); cy.ensureTextNotPresent(domainDescription); }); -}); \ No newline at end of file +});