From 888a1de9fc85169ead4eb9ba207ce85b56abcbd4 Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Tue, 2 Apr 2024 19:36:52 -0500 Subject: [PATCH] feat(lineage): give via and paths in entity lineage response (#10192) --- .../resolvers/load/EntityLineageResultResolver.java | 7 +++++++ .../datahub/graphql/types/mappers/MapperUtils.java | 9 +++++++++ .../mappers/UrnSearchAcrossLineageResultsMapper.java | 10 +--------- datahub-graphql-core/src/main/resources/entity.graphql | 4 ++++ datahub-graphql-core/src/main/resources/search.graphql | 5 +++++ .../metadata/graph/elastic/ESGraphQueryDAO.java | 8 +++++--- .../linkedin/metadata/search/LineageSearchService.java | 1 + .../linkedin/metadata/graph/LineageRelationship.pdl | 5 +++++ .../linkedin/metadata/search/LineageSearchEntity.pdl | 5 +++++ .../com.linkedin.entity.entities.snapshot.json | 5 +++++ .../com.linkedin.lineage.relationships.snapshot.json | 5 +++++ 11 files changed, 52 insertions(+), 12 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java index 8de18ec01e6dc2..e28ec3dbb870fa 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java @@ -1,8 +1,10 @@ package com.linkedin.datahub.graphql.resolvers.load; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.datahub.graphql.types.mappers.MapperUtils.*; import com.datahub.authorization.AuthorizationConfiguration; +import com.linkedin.common.UrnArrayArray; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.SetMode; @@ -156,6 +158,11 @@ private LineageRelationship mapEntityRelationship( result.setUpdatedActor(UrnToEntityMapper.map(context, updatedActor)); } result.setIsManual(lineageRelationship.hasIsManual() && lineageRelationship.isIsManual()); + if (lineageRelationship.getPaths() != null) { + UrnArrayArray paths = lineageRelationship.getPaths(); + result.setPaths( + paths.stream().map(path -> mapPath(context, path)).collect(Collectors.toList())); + } return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java index 3cae0155a86db5..6bda333256a4c9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java @@ -3,9 +3,11 @@ import static com.linkedin.datahub.graphql.util.SearchInsightsUtil.*; import static com.linkedin.metadata.utils.SearchUtil.*; +import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.AggregationMetadata; +import com.linkedin.datahub.graphql.generated.EntityPath; import com.linkedin.datahub.graphql.generated.FacetMetadata; import com.linkedin.datahub.graphql.generated.MatchedField; import com.linkedin.datahub.graphql.generated.SearchResult; @@ -104,4 +106,11 @@ public static SearchSuggestion mapSearchSuggestion( return new SearchSuggestion( suggestion.getText(), suggestion.getScore(), Math.toIntExact(suggestion.getFrequency())); } + + public static EntityPath mapPath(@Nullable final QueryContext context, UrnArray path) { + EntityPath entityPath = new EntityPath(); + entityPath.setPath( + path.stream().map(p -> UrnToEntityMapper.map(context, p)).collect(Collectors.toList())); + return entityPath; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java index b85303909c0801..ca363deb90c4de 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java @@ -3,11 +3,9 @@ import static com.linkedin.datahub.graphql.types.mappers.MapperUtils.*; import static com.linkedin.datahub.graphql.util.SearchInsightsUtil.*; -import com.linkedin.common.UrnArray; import com.linkedin.data.template.RecordTemplate; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Entity; -import com.linkedin.datahub.graphql.generated.EntityPath; import com.linkedin.datahub.graphql.generated.FreshnessStats; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResult; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResults; @@ -72,13 +70,7 @@ private SearchAcrossLineageResult mapResult( .setDegree(searchEntity.getDegree()) .setDegrees(new ArrayList<>(searchEntity.getDegrees())) .setExplored(Boolean.TRUE.equals(searchEntity.isExplored())) + .setIgnoredAsHop(Boolean.TRUE.equals(searchEntity.isIgnoredAsHop())) .build(); } - - private EntityPath mapPath(@Nullable final QueryContext context, UrnArray path) { - EntityPath entityPath = new EntityPath(); - entityPath.setPath( - path.stream().map(p -> UrnToEntityMapper.map(context, p)).collect(Collectors.toList())); - return entityPath; - } } diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index b750d206261018..106148c425791c 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -1331,6 +1331,10 @@ type LineageRelationship { """ isManual: Boolean + """ + The paths traversed for this relationship + """ + paths: [EntityPath] } """ diff --git a/datahub-graphql-core/src/main/resources/search.graphql b/datahub-graphql-core/src/main/resources/search.graphql index 13c1ff2e8a7648..499ac3a0860d40 100644 --- a/datahub-graphql-core/src/main/resources/search.graphql +++ b/datahub-graphql-core/src/main/resources/search.graphql @@ -747,6 +747,11 @@ type SearchAcrossLineageResult { """ explored: Boolean! + """ + Whether this relationship was ignored as a hop + """ + ignoredAsHop: Boolean! + } """ diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java index ea8d8fea54633c..bdcbf020ecf781 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java @@ -389,9 +389,11 @@ private Stream processOneHopLineage( || platformMatches( lineageRelationship.getEntity(), ignoreAsHops.get(entityType))))) - .map(LineageRelationship::getEntity) - .forEach(additionalCurrentLevel::add); - ; + .forEach( + lineageRelationship -> { + additionalCurrentLevel.add(lineageRelationship.getEntity()); + lineageRelationship.setIgnoredAsHop(true); + }); if (!additionalCurrentLevel.isEmpty()) { Stream ignoreAsHopUrns = processOneHopLineage( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java index bb316f6f2b41c3..94f56fec2acc93 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java @@ -739,6 +739,7 @@ private LineageSearchEntity buildLineageSearchEntity( entity.setDegrees(lineageRelationship.getDegrees()); } entity.setExplored(Boolean.TRUE.equals(lineageRelationship.isExplored())); + entity.setIgnoredAsHop(Boolean.TRUE.equals(lineageRelationship.isIgnoredAsHop())); } return entity; } diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/LineageRelationship.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/LineageRelationship.pdl index a169157955e67b..552dd7323b5517 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/LineageRelationship.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/LineageRelationship.pdl @@ -72,4 +72,9 @@ record LineageRelationship { * Marks this relationship as explored during the graph walk */ explored: optional boolean + + /** + * Whether this relationship was ignored as a hop while performing the graph walk + */ + ignoredAsHop: optional boolean } diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/search/LineageSearchEntity.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/search/LineageSearchEntity.pdl index fdfc8b2d53291c..3fd8a48c6bf5ee 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/search/LineageSearchEntity.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/search/LineageSearchEntity.pdl @@ -34,4 +34,9 @@ record LineageSearchEntity includes SearchEntity { * Marks an entity as having been explored for as a part of the graph walk */ explored: optional boolean + + /** + * Whether this relationship was ignored as a hop while performing the graph walk + */ + ignoredAsHop: optional boolean } \ No newline at end of file diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index 4915f06ffe5d2a..43845a5fbbf6a3 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -6210,6 +6210,11 @@ "type" : "boolean", "doc" : "Marks an entity as having been explored for as a part of the graph walk", "optional" : true + }, { + "name" : "ignoredAsHop", + "type" : "boolean", + "doc" : "Whether this relationship was ignored as a hop while performing the graph walk", + "optional" : true } ] } }, diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json index 00b3c925d0e731..3886faffadedbb 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json @@ -182,6 +182,11 @@ "type" : "boolean", "doc" : "Marks this relationship as explored during the graph walk", "optional" : true + }, { + "name" : "ignoredAsHop", + "type" : "boolean", + "doc" : "Whether this relationship was ignored as a hop while performing the graph walk", + "optional" : true } ] } },