diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 1ae3edae7aa90b..7da20ece44f6d6 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -45,7 +45,8 @@ jobs: "Salman-Apptware", "mayurinehate", "noggi", - "skrydal" + "skrydal", + "kevinkarchacryl" ]'), github.actor ) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java index a17745948eb823..88ac29b72dee83 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java @@ -1,19 +1,14 @@ package com.linkedin.datahub.graphql.analytics.service; +import static com.linkedin.metadata.Constants.CORP_USER_EDITABLE_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME; import static com.linkedin.metadata.Constants.CORP_USER_INFO_ASPECT_NAME; import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.dashboard.DashboardInfo; -import com.linkedin.datahub.graphql.generated.BarSegment; -import com.linkedin.datahub.graphql.generated.Cell; -import com.linkedin.datahub.graphql.generated.Entity; -import com.linkedin.datahub.graphql.generated.EntityProfileParams; -import com.linkedin.datahub.graphql.generated.LinkParams; -import com.linkedin.datahub.graphql.generated.NamedBar; -import com.linkedin.datahub.graphql.generated.Row; -import com.linkedin.datahub.graphql.generated.SearchParams; +import com.linkedin.datahub.graphql.generated.*; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.dataplatform.DataPlatformInfo; import com.linkedin.dataset.DatasetProperties; @@ -22,6 +17,7 @@ import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.client.EntityClient; import com.linkedin.glossary.GlossaryTermInfo; +import com.linkedin.identity.CorpUserEditableInfo; import com.linkedin.identity.CorpUserInfo; import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.GlossaryTermKey; @@ -35,6 +31,7 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; @@ -169,36 +166,79 @@ public static void convertToUserInfoRows( final Map gmsResponseByUser = entityClient.batchGetV2( opContext, - CORP_USER_INFO_ASPECT_NAME, + CORP_USER_ENTITY_NAME, userUrns, - ImmutableSet.of(CORP_USER_INFO_ASPECT_NAME)); - final Map urnToCorpUserInfo = + ImmutableSet.of(CORP_USER_INFO_ASPECT_NAME, CORP_USER_EDITABLE_INFO_ASPECT_NAME)); + final Stream> entityStream = gmsResponseByUser.entrySet().stream() .filter( entry -> entry.getValue() != null - && entry.getValue().getAspects().containsKey(CORP_USER_INFO_ASPECT_NAME)) - .collect( - Collectors.toMap( - Map.Entry::getKey, - entry -> + && (entry.getValue().getAspects().containsKey(CORP_USER_INFO_ASPECT_NAME) + || entry + .getValue() + .getAspects() + .containsKey(CORP_USER_EDITABLE_INFO_ASPECT_NAME))); + final Map> urnToCorpUserInfo = + entityStream.collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> { + CorpUserInfo userInfo = null; + CorpUserEditableInfo editableInfo = null; + try { + userInfo = new CorpUserInfo( entry .getValue() .getAspects() .get(CORP_USER_INFO_ASPECT_NAME) .getValue() - .data()))); + .data()); + } catch (Exception e) { + // nothing to do + } + try { + + editableInfo = + new CorpUserEditableInfo( + entry + .getValue() + .getAspects() + .get(CORP_USER_EDITABLE_INFO_ASPECT_NAME) + .getValue() + .data()); + } catch (Exception e) { + // nothing to do + } + + return Pair.of(userInfo, editableInfo); + })); // Populate a row with the user link, title, and email. rows.forEach( row -> { Urn urn = UrnUtils.getUrn(row.getCells().get(0).getValue()); EntityResponse response = gmsResponseByUser.get(urn); String maybeDisplayName = response != null ? getUserName(response).orElse(null) : null; - String maybeEmail = - urnToCorpUserInfo.containsKey(urn) ? urnToCorpUserInfo.get(urn).getEmail() : null; - String maybeTitle = - urnToCorpUserInfo.containsKey(urn) ? urnToCorpUserInfo.get(urn).getTitle() : null; + String maybeEmail = null; + String maybeTitle = null; + if (urnToCorpUserInfo.containsKey(urn)) { + Pair pair = urnToCorpUserInfo.get(urn); + if (pair.getLeft() != null) { + CorpUserInfo userInfo = pair.getLeft(); + maybeEmail = userInfo.getEmail(); + maybeTitle = userInfo.getTitle(); + } + if (pair.getRight() != null) { + CorpUserEditableInfo userInfo = pair.getRight(); + if (maybeEmail == null) { + maybeEmail = userInfo.getEmail(); + } + if (maybeTitle == null) { + maybeTitle = userInfo.getTitle(); + } + } + } if (maybeDisplayName != null) { row.getCells().get(0).setValue(maybeDisplayName); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/AnalyticsUtilTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/AnalyticsUtilTest.java new file mode 100644 index 00000000000000..ab1140d2380315 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/AnalyticsUtilTest.java @@ -0,0 +1,108 @@ +package com.linkedin.datahub.graphql.utils; + +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.DataMap; +import com.linkedin.datahub.graphql.analytics.service.AnalyticsUtil; +import com.linkedin.datahub.graphql.generated.Cell; +import com.linkedin.datahub.graphql.generated.Row; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.identity.CorpUserEditableInfo; +import com.linkedin.identity.CorpUserInfo; +import com.linkedin.metadata.Constants; +import io.datahubproject.metadata.context.OperationContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class AnalyticsUtilTest { + + @Mock private OperationContext mockOpContext; + + @Mock private EntityClient mockEntityClient; + + final String TEST_CORP_USER_INFO_TEST_USER = "Corp User"; + final String TEST_CORP_USER_EDITABLE_INFO_TEST_TITLE = "Editable Info Title"; + final String TEST_CORP_USER_EDITABLE_INFO_TEST_EMAIL = "Editable Info Email"; + + @BeforeMethod + public void setUp() { + MockitoAnnotations.openMocks(this); + } + + @Test + public void testConvertToUserInfoRows() throws Exception { + List rows = new ArrayList<>(); + rows.add(new Row(null, Arrays.asList(new Cell("urn:li:corpuser:testuser", null, null)))); + + // create a CorpUserInfo with only display name set + CorpUserInfo corpUserInfo = new CorpUserInfo(); + corpUserInfo.setActive(true); + corpUserInfo.setDisplayName(TEST_CORP_USER_INFO_TEST_USER); + + // create an editableInfo with the email and title set + CorpUserEditableInfo corpUserEditableInfo = new CorpUserEditableInfo(); + corpUserEditableInfo.setEmail(TEST_CORP_USER_EDITABLE_INFO_TEST_EMAIL); // Overriding email + corpUserEditableInfo.setTitle(TEST_CORP_USER_EDITABLE_INFO_TEST_TITLE); // Overriding title + + DataMap corpUserInfoDataMap = new DataMap(); + corpUserInfoDataMap.put("name", Constants.CORP_USER_INFO_ASPECT_NAME); + corpUserInfoDataMap.put("type", "VERSIONED"); + corpUserInfoDataMap.put("value", corpUserInfo.data()); + + DataMap corpUserEditableInfoDataMap = new DataMap(); + corpUserEditableInfoDataMap.put("name", Constants.CORP_USER_EDITABLE_INFO_ASPECT_NAME); + corpUserEditableInfoDataMap.put("type", "VERSIONED"); + corpUserEditableInfoDataMap.put("value", corpUserEditableInfo.data()); + + EnvelopedAspect corpUserInfoEnvelopedAspect = new EnvelopedAspect(corpUserInfoDataMap); + EnvelopedAspect corpUserEditableInfoEnvelopedAspect = + new EnvelopedAspect(corpUserEditableInfoDataMap); + + EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); + aspectMap.put(Constants.CORP_USER_INFO_ASPECT_NAME, corpUserInfoEnvelopedAspect); + aspectMap.put( + Constants.CORP_USER_EDITABLE_INFO_ASPECT_NAME, corpUserEditableInfoEnvelopedAspect); + + EntityResponse entityResponse = new EntityResponse(); + entityResponse.setAspects(aspectMap); + + Map entityResponseMap = new HashMap<>(); + Urn userUrn = UrnUtils.getUrn("urn:li:corpuser:testuser"); + entityResponseMap.put(userUrn, entityResponse); + + // method of the entity client we need to mock to retrieve the response map + when(mockEntityClient.batchGetV2( + eq(mockOpContext), eq(Constants.CORP_USER_ENTITY_NAME), anySet(), anySet())) + .thenReturn(entityResponseMap); + + // function we are testing + AnalyticsUtil.convertToUserInfoRows(mockOpContext, mockEntityClient, rows); + + Row updatedRow = rows.get(0); + List updatedCells = updatedRow.getCells(); + + // asserting that the display user is from CorpUserInfo and email, title are from EditableInfo + assertEquals(updatedCells.get(0).getValue(), TEST_CORP_USER_INFO_TEST_USER); + assertEquals( + updatedCells.get(1).getValue(), + TEST_CORP_USER_EDITABLE_INFO_TEST_TITLE); // Overriding title + assertEquals( + updatedCells.get(2).getValue(), + TEST_CORP_USER_EDITABLE_INFO_TEST_EMAIL); // Overriding email + } +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx index fe2a8c51f9377b..8ee0f637094d6d 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx @@ -50,26 +50,26 @@ export const Editor = forwardRef((props: EditorProps, ref) => { const { manager, state, getContext } = useRemirror({ extensions: () => [ new BlockquoteExtension(), - new BoldExtension(), - new BulletListExtension(), + new BoldExtension({}), + new BulletListExtension({}), new CodeBlockExtension({ syntaxTheme: 'base16_ateliersulphurpool_light' }), new CodeExtension(), - new DataHubMentionsExtension(), - new DropCursorExtension(), + new DataHubMentionsExtension({}), + new DropCursorExtension({}), new HardBreakExtension(), - new HeadingExtension(), - new HistoryExtension(), - new HorizontalRuleExtension(), + new HeadingExtension({}), + new HistoryExtension({}), + new HorizontalRuleExtension({}), new ImageExtension({ enableResizing: !readOnly }), new ItalicExtension(), new LinkExtension({ autoLink: true, defaultTarget: '_blank' }), - new ListItemExtension(), + new ListItemExtension({}), new MarkdownExtension({ htmlSanitizer: DOMPurify.sanitize, htmlToMarkdown, markdownToHtml }), new OrderedListExtension(), new UnderlineExtension(), new StrikeExtension(), new TableExtension({ resizable: false }), - ...(readOnly ? [] : [new HistoryExtension()]), + ...(readOnly ? [] : [new HistoryExtension({})]), ], content, stringHandler: 'markdown', diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock index 4e457abd50af7a..9c82b27c3b61f3 100644 --- a/docs-website/yarn.lock +++ b/docs-website/yarn.lock @@ -1827,7 +1827,7 @@ "@docusaurus/theme-search-algolia" "2.4.3" "@docusaurus/types" "2.4.3" -"@docusaurus/react-loadable@5.5.2", "react-loadable@npm:@docusaurus/react-loadable@5.5.2": +"@docusaurus/react-loadable@5.5.2": version "5.5.2" resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce" integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== @@ -4757,9 +4757,9 @@ cross-fetch@^3.1.5: node-fetch "^2.6.12" cross-spawn@^7.0.3: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" @@ -9713,6 +9713,14 @@ react-loadable-ssr-addon-v5-slorber@^1.0.1: dependencies: "@babel/runtime" "^7.10.3" +"react-loadable@npm:@docusaurus/react-loadable@5.5.2": + version "5.5.2" + resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce" + integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== + dependencies: + "@types/react" "*" + prop-types "^15.6.2" + react-markdown@^8.0.6: version "8.0.7" resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-8.0.7.tgz#c8dbd1b9ba5f1c5e7e5f2a44de465a3caafdf89b" diff --git a/metadata-ingestion/docs/sources/kafka/kafka.md b/metadata-ingestion/docs/sources/kafka/kafka.md index f9b50c13f02778..525f985be4662a 100644 --- a/metadata-ingestion/docs/sources/kafka/kafka.md +++ b/metadata-ingestion/docs/sources/kafka/kafka.md @@ -102,7 +102,29 @@ source: connection: bootstrap: "broker:9092" schema_registry_url: http://localhost:8081 +``` + +### OAuth Callback +The OAuth callback function can be set up using `config.connection.consumer_config.oauth_cb`. + +You need to specify a Python function reference in the format <python-module>:<function-name>. + +For example, in the configuration `oauth:create_token`, `create_token` is a function defined in `oauth.py`, and `oauth.py` must be accessible in the PYTHONPATH. +```YAML +source: + type: "kafka" + config: + # Set the custom schema registry implementation class + schema_registry_class: "datahub.ingestion.source.confluent_schema_registry.ConfluentSchemaRegistry" + # Coordinates + connection: + bootstrap: "broker:9092" + schema_registry_url: http://localhost:8081 + consumer_config: + security.protocol: "SASL_PLAINTEXT" + sasl.mechanism: "OAUTHBEARER" + oauth_cb: "oauth:create_token" # sink configs ``` diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index c6530c51c949d0..8ae112c0ab0b2b 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -741,8 +741,8 @@ "hive = datahub.ingestion.source.sql.hive:HiveSource", "hive-metastore = datahub.ingestion.source.sql.hive_metastore:HiveMetastoreSource", "json-schema = datahub.ingestion.source.schema.json_schema:JsonSchemaSource", - "kafka = datahub.ingestion.source.kafka:KafkaSource", - "kafka-connect = datahub.ingestion.source.kafka_connect:KafkaConnectSource", + "kafka = datahub.ingestion.source.kafka.kafka:KafkaSource", + "kafka-connect = datahub.ingestion.source.kafka.kafka_connect:KafkaConnectSource", "ldap = datahub.ingestion.source.ldap:LDAPSource", "looker = datahub.ingestion.source.looker.looker_source:LookerDashboardSource", "lookml = datahub.ingestion.source.looker.lookml_source:LookMLSource", diff --git a/metadata-ingestion/src/datahub/configuration/kafka.py b/metadata-ingestion/src/datahub/configuration/kafka.py index 07e2f759bb3ff6..b8d9ff994a51ab 100644 --- a/metadata-ingestion/src/datahub/configuration/kafka.py +++ b/metadata-ingestion/src/datahub/configuration/kafka.py @@ -1,6 +1,7 @@ from pydantic import Field, validator -from datahub.configuration.common import ConfigModel +from datahub.configuration.common import ConfigModel, ConfigurationError +from datahub.configuration.kafka_consumer_config import CallableConsumerConfig from datahub.configuration.validate_host_port import validate_host_port @@ -36,6 +37,16 @@ class KafkaConsumerConnectionConfig(_KafkaConnectionConfig): description="Extra consumer config serialized as JSON. These options will be passed into Kafka's DeserializingConsumer. See https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#deserializingconsumer and https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md .", ) + @validator("consumer_config") + @classmethod + def resolve_callback(cls, value: dict) -> dict: + if CallableConsumerConfig.is_callable_config(value): + try: + value = CallableConsumerConfig(value).callable_config() + except Exception as e: + raise ConfigurationError(e) + return value + class KafkaProducerConnectionConfig(_KafkaConnectionConfig): """Configuration class for holding connectivity information for Kafka producers""" diff --git a/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py b/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py new file mode 100644 index 00000000000000..d3ff5998d3e790 --- /dev/null +++ b/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py @@ -0,0 +1,35 @@ +import logging +from typing import Any, Dict, Optional + +from datahub.ingestion.api.registry import import_path + +logger = logging.getLogger(__name__) + + +class CallableConsumerConfig: + CALLBACK_ATTRIBUTE: str = "oauth_cb" + + def __init__(self, config: Dict[str, Any]): + self._config = config + + self._resolve_oauth_callback() + + def callable_config(self) -> Dict[str, Any]: + return self._config + + @staticmethod + def is_callable_config(config: Dict[str, Any]) -> bool: + return CallableConsumerConfig.CALLBACK_ATTRIBUTE in config + + def get_call_back_attribute(self) -> Optional[str]: + return self._config.get(CallableConsumerConfig.CALLBACK_ATTRIBUTE) + + def _resolve_oauth_callback(self) -> None: + if not self.get_call_back_attribute(): + return + + call_back = self.get_call_back_attribute() + + assert call_back # to silent lint + # Set the callback + self._config[CallableConsumerConfig.CALLBACK_ATTRIBUTE] = import_path(call_back) diff --git a/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py b/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py index ed51487ea6dab2..2b75d0dca53cb7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py +++ b/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py @@ -16,8 +16,10 @@ from datahub.ingestion.extractor import protobuf_util, schema_util from datahub.ingestion.extractor.json_schema_util import JsonSchemaTranslator from datahub.ingestion.extractor.protobuf_util import ProtobufSchema -from datahub.ingestion.source.kafka import KafkaSourceConfig, KafkaSourceReport -from datahub.ingestion.source.kafka_schema_registry_base import KafkaSchemaRegistryBase +from datahub.ingestion.source.kafka.kafka import KafkaSourceConfig, KafkaSourceReport +from datahub.ingestion.source.kafka.kafka_schema_registry_base import ( + KafkaSchemaRegistryBase, +) from datahub.metadata.com.linkedin.pegasus2avro.schema import ( KafkaSchema, SchemaField, diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py similarity index 97% rename from metadata-ingestion/src/datahub/ingestion/source/kafka.py rename to metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py index a757250a0d6c85..06d929774240ba 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py @@ -18,6 +18,7 @@ from datahub.configuration.common import AllowDenyPattern from datahub.configuration.kafka import KafkaConsumerConnectionConfig +from datahub.configuration.kafka_consumer_config import CallableConsumerConfig from datahub.configuration.source_common import ( DatasetSourceConfigMixin, LowerCaseDatasetUrnConfigMixin, @@ -49,7 +50,9 @@ ) from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.common.subtypes import DatasetSubTypes -from datahub.ingestion.source.kafka_schema_registry_base import KafkaSchemaRegistryBase +from datahub.ingestion.source.kafka.kafka_schema_registry_base import ( + KafkaSchemaRegistryBase, +) from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalHandler, StaleEntityRemovalSourceReport, @@ -143,7 +146,7 @@ class KafkaSourceConfig( def get_kafka_consumer( connection: KafkaConsumerConnectionConfig, ) -> confluent_kafka.Consumer: - return confluent_kafka.Consumer( + consumer = confluent_kafka.Consumer( { "group.id": "test", "bootstrap.servers": connection.bootstrap, @@ -151,6 +154,13 @@ def get_kafka_consumer( } ) + if CallableConsumerConfig.is_callable_config(connection.consumer_config): + # As per documentation, we need to explicitly call the poll method to make sure OAuth callback gets executed + # https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#kafka-client-configuration + consumer.poll(timeout=30) + + return consumer + @dataclass class KafkaSourceReport(StaleEntityRemovalSourceReport): diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_connect.py similarity index 100% rename from metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py rename to metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_connect.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_schema_registry_base.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_schema_registry_base.py similarity index 100% rename from metadata-ingestion/src/datahub/ingestion/source/kafka_schema_registry_base.py rename to metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_schema_registry_base.py diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py index a1b850b56677a4..ec7dbc8251b200 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py @@ -21,6 +21,9 @@ # See more below: # https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/acreldb/n0ejgx4895bofnn14rlguktfx5r3.htm "teradata", + # For SQL server, the default collation rules mean that all identifiers (schema, table, column names) + # are case preserving but case insensitive. + "mssql", } DIALECTS_WITH_DEFAULT_UPPERCASE_COLS = { # In some dialects, column identifiers are effectively case insensitive @@ -28,6 +31,9 @@ # automatically lowercase unquoted identifiers. "snowflake", } +assert DIALECTS_WITH_DEFAULT_UPPERCASE_COLS.issubset( + DIALECTS_WITH_CASE_INSENSITIVE_COLS +) class QueryType(enum.Enum): diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py index 4d0f9f7d8cf59d..9adb792a4be518 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py @@ -5,7 +5,7 @@ import logging import traceback from collections import defaultdict -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Set, Tuple, TypeVar, Union import pydantic.dataclasses import sqlglot @@ -873,6 +873,49 @@ def _translate_internal_column_lineage( ) +_StrOrNone = TypeVar("_StrOrNone", str, Optional[str]) + + +def _normalize_db_or_schema( + db_or_schema: _StrOrNone, + dialect: sqlglot.Dialect, +) -> _StrOrNone: + if db_or_schema is None: + return None + + # In snowflake, table identifiers must be uppercased to match sqlglot's behavior. + if is_dialect_instance(dialect, "snowflake"): + return db_or_schema.upper() + + # In mssql, table identifiers must be lowercased. + elif is_dialect_instance(dialect, "mssql"): + return db_or_schema.lower() + + return db_or_schema + + +def _simplify_select_into(statement: sqlglot.exp.Expression) -> sqlglot.exp.Expression: + """ + Check if the expression is a SELECT INTO statement. If so, converts it into a CTAS. + Other expressions are returned as-is. + """ + + if not (isinstance(statement, sqlglot.exp.Select) and statement.args.get("into")): + return statement + + # Convert from SELECT INTO + # to CREATE TABLE AS SELECT + into_expr: sqlglot.exp.Into = statement.args["into"].pop() + into_table = into_expr.this + + create = sqlglot.exp.Create( + this=into_table, + kind="TABLE", + expression=statement, + ) + return create + + def _sqlglot_lineage_inner( sql: sqlglot.exp.ExpOrStr, schema_resolver: SchemaResolverInterface, @@ -885,12 +928,9 @@ def _sqlglot_lineage_inner( else: dialect = get_dialect(default_dialect) - if is_dialect_instance(dialect, "snowflake"): - # in snowflake, table identifiers must be uppercased to match sqlglot's behavior. - if default_db: - default_db = default_db.upper() - if default_schema: - default_schema = default_schema.upper() + default_db = _normalize_db_or_schema(default_db, dialect) + default_schema = _normalize_db_or_schema(default_schema, dialect) + if is_dialect_instance(dialect, "redshift") and not default_schema: # On Redshift, there's no "USE SCHEMA " command. The default schema # is public, and "current schema" is the one at the front of the search path. @@ -918,6 +958,8 @@ def _sqlglot_lineage_inner( # original_statement.sql(pretty=True, dialect=dialect), # ) + statement = _simplify_select_into(statement) + # Make sure the tables are resolved with the default db / schema. # This only works for Unionable statements. For other types of statements, # we have to do it manually afterwards, but that's slightly lower accuracy diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py index 5413161d0b7a3d..bd98557e08aacc 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py @@ -61,7 +61,7 @@ def is_dialect_instance( else: platforms = list(platforms) - dialects = [sqlglot.Dialect.get_or_raise(platform) for platform in platforms] + dialects = [get_dialect(platform) for platform in platforms] if any(isinstance(dialect, dialect_class.__class__) for dialect_class in dialects): return True diff --git a/metadata-ingestion/src/datahub/utilities/file_backed_collections.py b/metadata-ingestion/src/datahub/utilities/file_backed_collections.py index b67f9c1a9ffa15..b0f5022446de15 100644 --- a/metadata-ingestion/src/datahub/utilities/file_backed_collections.py +++ b/metadata-ingestion/src/datahub/utilities/file_backed_collections.py @@ -228,6 +228,12 @@ def __post_init__(self) -> None: else: self._conn = ConnectionWrapper() + if sqlite3.sqlite_version_info < (3, 24, 0): + # We use the ON CONFLICT clause to implement UPSERTs with sqlite. + # This was added in 3.24.0 from 2018-06-04. + # See https://www.sqlite.org/lang_conflict.html + raise RuntimeError("SQLite version 3.24.0 or later is required") + # We keep a small cache in memory to avoid having to serialize/deserialize # data from the database too often. We use an OrderedDict to build # a poor-man's LRU cache. diff --git a/metadata-ingestion/tests/integration/kafka/kafka_to_file_oauth.yml b/metadata-ingestion/tests/integration/kafka/kafka_to_file_oauth.yml new file mode 100644 index 00000000000000..34cf8fd47e658c --- /dev/null +++ b/metadata-ingestion/tests/integration/kafka/kafka_to_file_oauth.yml @@ -0,0 +1,20 @@ +run_id: kafka-test + +source: + type: kafka + config: + connection: + bootstrap: "localhost:29092" + schema_registry_url: "http://localhost:28081" + consumer_config: + security.protocol: "SASL_PLAINTEXT" + sasl.mechanism: "OAUTHBEARER" + oauth_cb: "oauth:create_token" + domain: + "urn:li:domain:sales": + allow: + - "key_value_topic" +sink: + type: file + config: + filename: "./kafka_mces.json" diff --git a/metadata-ingestion/tests/integration/kafka/oauth.py b/metadata-ingestion/tests/integration/kafka/oauth.py new file mode 100644 index 00000000000000..28cfee521d6c0f --- /dev/null +++ b/metadata-ingestion/tests/integration/kafka/oauth.py @@ -0,0 +1,14 @@ +import logging +from typing import Any, Tuple + +logger = logging.getLogger(__name__) + +MESSAGE: str = "OAuth token `create_token` callback" + + +def create_token(*args: Any, **kwargs: Any) -> Tuple[str, int]: + logger.warning(MESSAGE) + return ( + "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJjbGllbnRfaWQiOiJrYWZrYV9jbGllbnQiLCJleHAiOjE2OTg3NjYwMDB9.dummy_sig_abcdef123456", + 3600, + ) diff --git a/metadata-ingestion/tests/integration/kafka/test_kafka.py b/metadata-ingestion/tests/integration/kafka/test_kafka.py index dfdbea5de5cbfd..597889c8440b7a 100644 --- a/metadata-ingestion/tests/integration/kafka/test_kafka.py +++ b/metadata-ingestion/tests/integration/kafka/test_kafka.py @@ -1,10 +1,14 @@ +import logging import subprocess import pytest +import yaml from freezegun import freeze_time from datahub.ingestion.api.source import SourceCapability -from datahub.ingestion.source.kafka import KafkaSource +from datahub.ingestion.run.pipeline import Pipeline +from datahub.ingestion.source.kafka.kafka import KafkaSource +from tests.integration.kafka import oauth # type: ignore from tests.test_helpers import mce_helpers, test_connection_helpers from tests.test_helpers.click_helpers import run_datahub_cmd from tests.test_helpers.docker_helpers import wait_for_port @@ -99,3 +103,36 @@ def test_kafka_test_connection(mock_kafka_service, config_dict, is_success): SourceCapability.SCHEMA_METADATA: "Failed to establish a new connection" }, ) + + +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_kafka_oauth_callback( + mock_kafka_service, test_resources_dir, pytestconfig, tmp_path, mock_time +): + # Run the metadata ingestion pipeline. + config_file = (test_resources_dir / "kafka_to_file_oauth.yml").resolve() + + log_file = tmp_path / "kafka_oauth_message.log" + + file_handler = logging.FileHandler( + str(log_file) + ) # Add a file handler to later validate a test-case + logging.getLogger().addHandler(file_handler) + + recipe: dict = {} + with open(config_file) as fp: + recipe = yaml.safe_load(fp) + + pipeline = Pipeline.create(recipe) + + pipeline.run() + + is_found: bool = False + with open(log_file, "r") as file: + for line_number, line in enumerate(file, 1): + if oauth.MESSAGE in line: + is_found = True + break + + assert is_found diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json index 8e06ba9a80906b..5881ad5de0bcb8 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json @@ -1024,6 +1024,32 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD)", "type": "TRANSFORMED" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),client_director)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),month_wid)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV),cd_agent_key)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),client_manager_closing_month)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),month_wid)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV),agent_key)" + ], + "confidenceScore": 1.0 + } ] } }, diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json index 558548bfc7a698..4302c41140dc6c 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", + "job_id": "01afcab8-187c-459f-828e-727196a1832d", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-16 15:59:53.077000", - "date_modified": "2024-09-16 15:59:53.217000", + "date_created": "2024-11-21 21:01:26.550000", + "date_modified": "2024-11-21 21:01:26.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -154,6 +154,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", @@ -231,10 +247,15 @@ "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -245,17 +266,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -341,10 +357,15 @@ "entityType": "container", "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -355,17 +376,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", + "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -451,10 +467,15 @@ "entityType": "container", "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -465,17 +486,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", + "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -561,10 +577,15 @@ "entityType": "container", "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -575,17 +596,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", + "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -671,10 +687,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -685,17 +706,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", + "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -781,10 +797,15 @@ "entityType": "container", "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -795,17 +816,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", + "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -891,10 +907,15 @@ "entityType": "container", "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -905,17 +926,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", + "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1001,10 +1017,15 @@ "entityType": "container", "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1015,17 +1036,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", + "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1111,10 +1127,15 @@ "entityType": "container", "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1125,17 +1146,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", + "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1217,22 +1233,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", @@ -1386,6 +1386,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1459,22 +1475,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1943,15 +1943,13 @@ } }, { - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", "changeType": "UPSERT", - "aspectName": "dataFlowInfo", + "aspectName": "container", "aspect": { "json": { - "customProperties": {}, - "externalUrl": "", - "name": "DemoData.Foo.stored_procedures" + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" } }, "systemMetadata": { @@ -1961,9 +1959,188 @@ } }, { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", - "changeType": "UPSERT", + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "is_view": "True" + }, + "name": "PersonsView", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.PersonsView", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", + "changeType": "UPSERT", + "aspectName": "dataFlowInfo", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "", + "name": "DemoData.Foo.stored_procedures" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { "json": { @@ -1973,8 +2150,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-16 15:59:53.010000", - "date_modified": "2024-09-16 15:59:53.010000" + "date_created": "2024-11-21 21:01:26.483000", + "date_modified": "2024-11-21 21:01:26.483000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2007,6 +2184,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2084,10 +2277,15 @@ "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2098,17 +2296,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2194,10 +2387,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2208,17 +2406,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", + "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2300,22 +2493,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", @@ -2425,6 +2602,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", @@ -2502,10 +2695,15 @@ "entityType": "container", "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2516,17 +2714,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", + "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -2612,10 +2805,15 @@ "entityType": "container", "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2626,17 +2824,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", + "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -2722,10 +2915,15 @@ "entityType": "container", "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2736,17 +2934,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", + "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -2832,10 +3025,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2846,17 +3044,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", + "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -2942,10 +3135,15 @@ "entityType": "container", "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2956,17 +3154,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", + "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3052,10 +3245,15 @@ "entityType": "container", "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -3066,17 +3264,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", + "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3162,10 +3355,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -3176,17 +3374,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", + "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3272,10 +3465,15 @@ "entityType": "container", "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -3286,17 +3484,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", + "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3382,10 +3575,15 @@ "entityType": "container", "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -3396,17 +3594,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", + "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3488,22 +3681,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", @@ -3669,6 +3846,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", @@ -3742,22 +3935,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", @@ -4079,6 +4256,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", @@ -4156,10 +4349,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -4170,17 +4368,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", + "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -4266,10 +4459,15 @@ "entityType": "container", "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -4280,17 +4478,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", + "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -4376,10 +4569,15 @@ "entityType": "container", "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -4389,16 +4587,20 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "upstreamLineage", "aspect": { "json": { - "path": [ + "upstreams": [ { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "type": "VIEW" } ] } @@ -4472,5 +4674,21 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index 161d40ea91d91a..0a50556edc6388 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", + "job_id": "01afcab8-187c-459f-828e-727196a1832d", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-16 15:59:53.077000", - "date_modified": "2024-09-16 15:59:53.217000", + "date_created": "2024-11-21 21:01:26.550000", + "date_modified": "2024-11-21 21:01:26.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -154,6 +154,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", @@ -231,10 +247,15 @@ "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -245,17 +266,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -341,10 +357,15 @@ "entityType": "container", "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -355,17 +376,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", + "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -451,10 +467,15 @@ "entityType": "container", "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -465,17 +486,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", + "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -561,10 +577,15 @@ "entityType": "container", "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -575,17 +596,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", + "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -671,10 +687,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -685,17 +706,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", + "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -781,10 +797,15 @@ "entityType": "container", "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -795,17 +816,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", + "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -891,10 +907,15 @@ "entityType": "container", "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -905,17 +926,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", + "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1001,10 +1017,15 @@ "entityType": "container", "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1015,17 +1036,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", + "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1111,10 +1127,15 @@ "entityType": "container", "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1125,17 +1146,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", + "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1217,22 +1233,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", @@ -1386,6 +1386,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1459,22 +1475,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1942,6 +1942,183 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "is_view": "True" + }, + "name": "PersonsView", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.PersonsView", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", @@ -1973,8 +2150,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-16 15:59:53.010000", - "date_modified": "2024-09-16 15:59:53.010000" + "date_created": "2024-11-21 21:01:26.483000", + "date_modified": "2024-11-21 21:01:26.483000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2007,6 +2184,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2084,10 +2277,15 @@ "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2098,17 +2296,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2194,10 +2387,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2208,17 +2406,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", + "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2304,10 +2497,15 @@ "entityType": "container", "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2317,16 +2515,20 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "upstreamLineage", "aspect": { "json": { - "path": [ + "upstreams": [ { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "type": "VIEW" } ] } @@ -2400,5 +2602,21 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json index 161d40ea91d91a..0a50556edc6388 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", + "job_id": "01afcab8-187c-459f-828e-727196a1832d", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-16 15:59:53.077000", - "date_modified": "2024-09-16 15:59:53.217000", + "date_created": "2024-11-21 21:01:26.550000", + "date_modified": "2024-11-21 21:01:26.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -154,6 +154,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", @@ -231,10 +247,15 @@ "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -245,17 +266,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -341,10 +357,15 @@ "entityType": "container", "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -355,17 +376,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", + "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -451,10 +467,15 @@ "entityType": "container", "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -465,17 +486,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", + "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -561,10 +577,15 @@ "entityType": "container", "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -575,17 +596,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", + "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -671,10 +687,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -685,17 +706,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", + "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -781,10 +797,15 @@ "entityType": "container", "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -795,17 +816,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", + "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -891,10 +907,15 @@ "entityType": "container", "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -905,17 +926,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", + "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1001,10 +1017,15 @@ "entityType": "container", "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1015,17 +1036,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", + "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1111,10 +1127,15 @@ "entityType": "container", "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1125,17 +1146,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", + "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1217,22 +1233,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", @@ -1386,6 +1386,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1459,22 +1475,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1942,6 +1942,183 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "is_view": "True" + }, + "name": "PersonsView", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.PersonsView", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", @@ -1973,8 +2150,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-16 15:59:53.010000", - "date_modified": "2024-09-16 15:59:53.010000" + "date_created": "2024-11-21 21:01:26.483000", + "date_modified": "2024-11-21 21:01:26.483000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2007,6 +2184,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2084,10 +2277,15 @@ "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2098,17 +2296,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2194,10 +2387,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2208,17 +2406,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", + "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2304,10 +2497,15 @@ "entityType": "container", "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2317,16 +2515,20 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "upstreamLineage", "aspect": { "json": { - "path": [ + "upstreams": [ { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "type": "VIEW" } ] } @@ -2400,5 +2602,21 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json index 29124f6fc156c4..0279a94084ce56 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", + "job_id": "01afcab8-187c-459f-828e-727196a1832d", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-16 15:59:53.077000", - "date_modified": "2024-09-16 15:59:53.217000", + "date_created": "2024-11-21 21:01:26.550000", + "date_modified": "2024-11-21 21:01:26.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -154,6 +154,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", @@ -231,10 +247,15 @@ "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -245,17 +266,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -341,10 +357,15 @@ "entityType": "container", "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -355,17 +376,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", + "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -451,10 +467,15 @@ "entityType": "container", "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -465,17 +486,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", + "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -561,10 +577,15 @@ "entityType": "container", "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -575,17 +596,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", + "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -671,10 +687,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -685,17 +706,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", + "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -781,10 +797,15 @@ "entityType": "container", "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -795,17 +816,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", + "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -891,10 +907,15 @@ "entityType": "container", "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -905,17 +926,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", + "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1001,10 +1017,15 @@ "entityType": "container", "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1015,17 +1036,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", + "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1111,10 +1127,15 @@ "entityType": "container", "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1125,17 +1146,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", + "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1217,22 +1233,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", @@ -1386,6 +1386,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1459,22 +1475,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1942,6 +1942,183 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "is_view": "True" + }, + "name": "PersonsView", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "demodata.foo.personsview", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", @@ -1973,8 +2150,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-16 15:59:53.010000", - "date_modified": "2024-09-16 15:59:53.010000" + "date_created": "2024-11-21 21:01:26.483000", + "date_modified": "2024-11-21 21:01:26.483000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2007,6 +2184,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2084,10 +2277,15 @@ "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2098,17 +2296,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2194,10 +2387,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2208,17 +2406,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", + "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2304,10 +2497,15 @@ "entityType": "container", "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2317,16 +2515,66 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "upstreamLineage", "aspect": { "json": { - "path": [ + "upstreams": [ { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),Age)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),Age)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),FirstName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),FirstName)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),LastName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),LastName)" + ], + "confidenceScore": 1.0 } ] } diff --git a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql index 77ecabc5a3fffc..f495db3b91cfae 100644 --- a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql +++ b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql @@ -32,6 +32,8 @@ CREATE TABLE Foo.Persons ( Age int ); GO +CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons; +GO CREATE TABLE Foo.SalesReason ( TempID int NOT NULL, diff --git a/metadata-ingestion/tests/unit/api/test_pipeline.py b/metadata-ingestion/tests/unit/api/test_pipeline.py index 432d8e11c1c0b4..fe3d3160b729a1 100644 --- a/metadata-ingestion/tests/unit/api/test_pipeline.py +++ b/metadata-ingestion/tests/unit/api/test_pipeline.py @@ -33,7 +33,9 @@ class TestPipeline: @patch("confluent_kafka.Consumer", autospec=True) - @patch("datahub.ingestion.source.kafka.KafkaSource.get_workunits", autospec=True) + @patch( + "datahub.ingestion.source.kafka.kafka.KafkaSource.get_workunits", autospec=True + ) @patch("datahub.ingestion.sink.console.ConsoleSink.close", autospec=True) @freeze_time(FROZEN_TIME) def test_configure(self, mock_sink, mock_source, mock_consumer): @@ -198,7 +200,9 @@ def test_configure_with_rest_sink_with_additional_props_initializes_graph( assert pipeline.ctx.graph.config.token == pipeline.config.sink.config["token"] @freeze_time(FROZEN_TIME) - @patch("datahub.ingestion.source.kafka.KafkaSource.get_workunits", autospec=True) + @patch( + "datahub.ingestion.source.kafka.kafka.KafkaSource.get_workunits", autospec=True + ) def test_configure_with_file_sink_does_not_init_graph(self, mock_source, tmp_path): pipeline = Pipeline.create( { diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_casing_resolver.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_casing_resolver.json new file mode 100644 index 00000000000000..072260f5cd1651 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_casing_resolver.json @@ -0,0 +1,82 @@ +{ + "query_type": "SELECT", + "query_type_props": {}, + "query_fingerprint": "6a779a57ffb2598c301606d3a7d82142a7af8b102efa55a2c9a4e960fd55ac07", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "age", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)", + "column": "Age" + } + ] + }, + { + "downstream": { + "table": null, + "column": "name", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "native_column_type": "VARCHAR(16777216)" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)", + "column": "Name" + } + ] + }, + { + "downstream": { + "table": null, + "column": "uppercased_col", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "native_column_type": "VARCHAR(16777216)" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)", + "column": "Uppercased_Col" + } + ] + }, + { + "downstream": { + "table": null, + "column": "count", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [] + } + ], + "debug_info": { + "confidence": 0.9, + "generalized_statement": "SELECT Age, name, UPPERCASED_COL, COUNT(*) AS Count FROM Foo.Persons GROUP BY Age" + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_select_into.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_select_into.json new file mode 100644 index 00000000000000..ff8609922a9bd8 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_select_into.json @@ -0,0 +1,48 @@ +{ + "query_type": "SELECT", + "query_type_props": {}, + "query_fingerprint": "d18c43ed4d7f0e303b88c40ac078c5505be473edf0136e42a77dccc090d1eb8b", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)" + ], + "out_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)" + ], + "column_lineage": [ + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)", + "column": "AGE", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)", + "column": "Age" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)", + "column": "Count", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [] + } + ], + "debug_info": { + "confidence": 0.9, + "generalized_statement": "SELECT age AS AGE, COUNT(*) AS Count INTO Foo.age_dist FROM Foo.Persons GROUP BY Age" + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py index 170341230205f3..2df6f6fa26cb56 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py @@ -1279,3 +1279,47 @@ def test_sqlite_attach_database() -> None: expected_file=RESOURCE_DIR / "test_sqlite_attach_database.json", allow_table_error=True, ) + + +def test_mssql_casing_resolver() -> None: + assert_sql_result( + """\ +SELECT Age, name, UPPERCASED_COL, COUNT(*) as Count +FROM Foo.Persons +GROUP BY Age +""", + dialect="mssql", + default_db="NewData", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)": { + "Age": "INTEGER", + "Name": "VARCHAR(16777216)", + "Uppercased_Col": "VARCHAR(16777216)", + }, + }, + expected_file=RESOURCE_DIR / "test_mssql_casing_resolver.json", + ) + + +def test_mssql_select_into() -> None: + assert_sql_result( + """\ +SELECT age as AGE, COUNT(*) as Count +INTO Foo.age_dist +FROM Foo.Persons +GROUP BY Age +""", + dialect="mssql", + default_db="NewData", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)": { + "Age": "INTEGER", + "Name": "VARCHAR(16777216)", + }, + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)": { + "AGE": "INTEGER", + "Count": "INTEGER", + }, + }, + expected_file=RESOURCE_DIR / "test_mssql_select_into.json", + ) diff --git a/metadata-ingestion/tests/unit/test_confluent_schema_registry.py b/metadata-ingestion/tests/unit/test_confluent_schema_registry.py index b047cd16c52a97..3500636f00eddf 100644 --- a/metadata-ingestion/tests/unit/test_confluent_schema_registry.py +++ b/metadata-ingestion/tests/unit/test_confluent_schema_registry.py @@ -8,7 +8,7 @@ ) from datahub.ingestion.source.confluent_schema_registry import ConfluentSchemaRegistry -from datahub.ingestion.source.kafka import KafkaSourceConfig, KafkaSourceReport +from datahub.ingestion.source.kafka.kafka import KafkaSourceConfig, KafkaSourceReport class ConfluentSchemaRegistryTest(unittest.TestCase): diff --git a/metadata-ingestion/tests/unit/test_kafka_source.py b/metadata-ingestion/tests/unit/test_kafka_source.py index b4e37d288a3041..dfd32085b77055 100644 --- a/metadata-ingestion/tests/unit/test_kafka_source.py +++ b/metadata-ingestion/tests/unit/test_kafka_source.py @@ -23,7 +23,7 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.workunit import MetadataWorkUnit -from datahub.ingestion.source.kafka import KafkaSource, KafkaSourceConfig +from datahub.ingestion.source.kafka.kafka import KafkaSource, KafkaSourceConfig from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.schema_classes import ( BrowsePathsClass, @@ -38,11 +38,13 @@ @pytest.fixture def mock_admin_client(): - with patch("datahub.ingestion.source.kafka.AdminClient", autospec=True) as mock: + with patch( + "datahub.ingestion.source.kafka.kafka.AdminClient", autospec=True + ) as mock: yield mock -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_configuration(mock_kafka): ctx = PipelineContext(run_id="test") kafka_source = KafkaSource( @@ -53,7 +55,7 @@ def test_kafka_source_configuration(mock_kafka): assert mock_kafka.call_count == 1 -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_workunits_wildcard_topic(mock_kafka, mock_admin_client): mock_kafka_instance = mock_kafka.return_value mock_cluster_metadata = MagicMock() @@ -74,7 +76,7 @@ def test_kafka_source_workunits_wildcard_topic(mock_kafka, mock_admin_client): assert len(workunits) == 4 -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_workunits_topic_pattern(mock_kafka, mock_admin_client): mock_kafka_instance = mock_kafka.return_value mock_cluster_metadata = MagicMock() @@ -108,7 +110,7 @@ def test_kafka_source_workunits_topic_pattern(mock_kafka, mock_admin_client): assert len(workunits) == 4 -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_workunits_with_platform_instance(mock_kafka, mock_admin_client): PLATFORM_INSTANCE = "kafka_cluster" PLATFORM = "kafka" @@ -160,7 +162,7 @@ def test_kafka_source_workunits_with_platform_instance(mock_kafka, mock_admin_cl assert f"/prod/{PLATFORM}/{PLATFORM_INSTANCE}" in browse_path_aspects[0].paths -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_workunits_no_platform_instance(mock_kafka, mock_admin_client): PLATFORM = "kafka" TOPIC_NAME = "test" @@ -204,7 +206,7 @@ def test_kafka_source_workunits_no_platform_instance(mock_kafka, mock_admin_clie assert f"/prod/{PLATFORM}" in browse_path_aspects[0].paths -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_close(mock_kafka, mock_admin_client): mock_kafka_instance = mock_kafka.return_value ctx = PipelineContext(run_id="test") @@ -223,7 +225,7 @@ def test_close(mock_kafka, mock_admin_client): "datahub.ingestion.source.confluent_schema_registry.SchemaRegistryClient", autospec=True, ) -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_workunits_schema_registry_subject_name_strategies( mock_kafka_consumer, mock_schema_registry_client, mock_admin_client ): @@ -415,7 +417,7 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: "datahub.ingestion.source.confluent_schema_registry.SchemaRegistryClient", autospec=True, ) -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_ignore_warnings_on_schema_type( mock_kafka_consumer, mock_schema_registry_client, @@ -483,8 +485,8 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: assert kafka_source.report.warnings -@patch("datahub.ingestion.source.kafka.AdminClient", autospec=True) -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.AdminClient", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_succeeds_with_admin_client_init_error( mock_kafka, mock_kafka_admin_client ): @@ -513,8 +515,8 @@ def test_kafka_source_succeeds_with_admin_client_init_error( assert len(workunits) == 2 -@patch("datahub.ingestion.source.kafka.AdminClient", autospec=True) -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.AdminClient", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_succeeds_with_describe_configs_error( mock_kafka, mock_kafka_admin_client ): @@ -550,7 +552,7 @@ def test_kafka_source_succeeds_with_describe_configs_error( "datahub.ingestion.source.confluent_schema_registry.SchemaRegistryClient", autospec=True, ) -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_topic_meta_mappings( mock_kafka_consumer, mock_schema_registry_client, mock_admin_client ): diff --git a/smoke-test/tests/cypress/yarn.lock b/smoke-test/tests/cypress/yarn.lock index 2433e9f8fae08e..c6116609b11467 100644 --- a/smoke-test/tests/cypress/yarn.lock +++ b/smoke-test/tests/cypress/yarn.lock @@ -510,9 +510,9 @@ core-util-is@1.0.2: integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= cross-spawn@^7.0.0, cross-spawn@^7.0.2: - version "7.0.3" - resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0"