From 2115710e309bb5c5436d11a25c9f101d62c695ac Mon Sep 17 00:00:00 2001 From: Ethan Cartwright Date: Fri, 27 Oct 2023 13:05:30 -0400 Subject: [PATCH] add logic to account for non dbt upstreams --- .../metadata/kafka/hook/siblings/SiblingAssociationHook.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java index 7cbe53dee9fe4b..064f987ff1ba92 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java @@ -205,9 +205,8 @@ private void handleSourceDatasetEvent(MetadataChangeLog event, DatasetUrn source // We're assuming a data asset (eg. snowflake table) will only ever be downstream of 1 dbt model if (dbtUpstreams.size() == 1) { setSiblingsAndSoftDeleteSibling(dbtUpstreams.get(0).getDataset(), sourceUrn); - } else { + } else if (dbtUpstreams.size() > 1) { log.error("{} has an unexpected number of dbt upstreams: {}. Not adding any as siblings.", sourceUrn.toString(), dbtUpstreams.size()); - } } }