diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 88900e06d48451..1f0d1b409fe0b2 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -25,4 +25,4 @@ dependencies { compileOnly 'org.projectlombok:lombok:1.18.30' annotationProcessor 'org.projectlombok:lombok:1.18.30' -} \ No newline at end of file +} diff --git a/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java b/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java index e97acb0b43c81d..01d61b6119b0a2 100644 --- a/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java +++ b/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java @@ -60,6 +60,7 @@ public class OpenApiEntities { .add("dataProductProperties") .add("institutionalMemory") .add("forms").add("formInfo").add("dynamicFormAssignment") + .add("businessAttributeInfo") .build(); private final static ImmutableSet ENTITY_EXCLUSIONS = ImmutableSet.builder() diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index c8ebf4042182d8..3296853145a470 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -33,6 +33,8 @@ import com.linkedin.datahub.graphql.generated.BrowsePathEntry; import com.linkedin.datahub.graphql.generated.BrowseResultGroupV2; import com.linkedin.datahub.graphql.generated.BrowseResults; +import com.linkedin.datahub.graphql.generated.BusinessAttribute; +import com.linkedin.datahub.graphql.generated.BusinessAttributeAssociation; import com.linkedin.datahub.graphql.generated.Chart; import com.linkedin.datahub.graphql.generated.ChartInfo; import com.linkedin.datahub.graphql.generated.Container; @@ -69,6 +71,7 @@ import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata; import com.linkedin.datahub.graphql.generated.LineageRelationship; import com.linkedin.datahub.graphql.generated.ListAccessTokenResult; +import com.linkedin.datahub.graphql.generated.ListBusinessAttributesResult; import com.linkedin.datahub.graphql.generated.ListDomainsResult; import com.linkedin.datahub.graphql.generated.ListGroupsResult; import com.linkedin.datahub.graphql.generated.ListOwnershipTypesResult; @@ -116,6 +119,12 @@ import com.linkedin.datahub.graphql.resolvers.browse.BrowsePathsResolver; import com.linkedin.datahub.graphql.resolvers.browse.BrowseResolver; import com.linkedin.datahub.graphql.resolvers.browse.EntityBrowsePathsResolver; +import com.linkedin.datahub.graphql.resolvers.businessattribute.AddBusinessAttributeResolver; +import com.linkedin.datahub.graphql.resolvers.businessattribute.CreateBusinessAttributeResolver; +import com.linkedin.datahub.graphql.resolvers.businessattribute.DeleteBusinessAttributeResolver; +import com.linkedin.datahub.graphql.resolvers.businessattribute.ListBusinessAttributesResolver; +import com.linkedin.datahub.graphql.resolvers.businessattribute.RemoveBusinessAttributeResolver; +import com.linkedin.datahub.graphql.resolvers.businessattribute.UpdateBusinessAttributeResolver; import com.linkedin.datahub.graphql.resolvers.chart.BrowseV2Resolver; import com.linkedin.datahub.graphql.resolvers.chart.ChartStatsSummaryResolver; import com.linkedin.datahub.graphql.resolvers.config.AppConfigResolver; @@ -292,6 +301,7 @@ import com.linkedin.datahub.graphql.types.aspect.AspectType; import com.linkedin.datahub.graphql.types.assertion.AssertionType; import com.linkedin.datahub.graphql.types.auth.AccessTokenMetadataType; +import com.linkedin.datahub.graphql.types.businessattribute.BusinessAttributeType; import com.linkedin.datahub.graphql.types.chart.ChartType; import com.linkedin.datahub.graphql.types.common.mappers.OperationMapper; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; @@ -350,6 +360,7 @@ import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; import com.linkedin.metadata.recommendation.RecommendationsService; +import com.linkedin.metadata.service.BusinessAttributeService; import com.linkedin.metadata.service.DataProductService; import com.linkedin.metadata.service.ERModelRelationshipService; import com.linkedin.metadata.service.FormService; @@ -428,6 +439,7 @@ public class GmsGraphQLEngine { private final FormService formService; private final RestrictedService restrictedService; + private final BusinessAttributeService businessAttributeService; private final FeatureFlags featureFlags; private final IngestionConfiguration ingestionConfiguration; @@ -485,6 +497,8 @@ public class GmsGraphQLEngine { private final int graphQLQueryComplexityLimit; private final int graphQLQueryDepthLimit; + private final BusinessAttributeType businessAttributeType; + /** A list of GraphQL Plugins that extend the core engine */ private final List graphQLPlugins; @@ -543,6 +557,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { this.formService = args.formService; this.restrictedService = args.restrictedService; + this.businessAttributeService = args.businessAttributeService; this.ingestionConfiguration = Objects.requireNonNull(args.ingestionConfiguration); this.authenticationConfiguration = Objects.requireNonNull(args.authenticationConfiguration); this.authorizationConfiguration = Objects.requireNonNull(args.authorizationConfiguration); @@ -597,6 +612,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { this.graphQLQueryComplexityLimit = args.graphQLQueryComplexityLimit; this.graphQLQueryDepthLimit = args.graphQLQueryDepthLimit; + this.businessAttributeType = new BusinessAttributeType(entityClient); // Init Lists this.entityTypes = ImmutableList.of( @@ -638,7 +654,8 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { entityTypeType, formType, incidentType, - restrictedType); + restrictedType, + businessAttributeType); this.loadableTypes = new ArrayList<>(entityTypes); // Extend loadable types with types from the plugins // This allows us to offer search and browse capabilities out of the box for those types @@ -731,6 +748,8 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureIncidentResolvers(builder); configureRestrictedResolvers(builder); configureRoleResolvers(builder); + configureBusinessAttributeResolver(builder); + configureBusinessAttributeAssociationResolver(builder); } private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) { @@ -1019,7 +1038,11 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { "listOwnershipTypes", new ListOwnershipTypesResolver(this.entityClient)) .dataFetcher( "browseV2", - new BrowseV2Resolver(this.entityClient, this.viewService, this.formService))); + new BrowseV2Resolver(this.entityClient, this.viewService, this.formService)) + .dataFetcher("businessAttribute", getResolver(businessAttributeType)) + .dataFetcher( + "listBusinessAttributes", + new ListBusinessAttributesResolver(this.entityClient))); } private DataFetcher getEntitiesResolver() { @@ -1072,194 +1095,210 @@ private String getUrnField(DataFetchingEnvironment env) { private void configureMutationResolvers(final RuntimeWiring.Builder builder) { builder.type( "Mutation", - typeWiring -> + typeWiring -> { + typeWiring + .dataFetcher("updateDataset", new MutableTypeResolver<>(datasetType)) + .dataFetcher("updateDatasets", new MutableTypeBatchResolver<>(datasetType)) + .dataFetcher( + "createTag", new CreateTagResolver(this.entityClient, this.entityService)) + .dataFetcher("updateTag", new MutableTypeResolver<>(tagType)) + .dataFetcher("setTagColor", new SetTagColorResolver(entityClient, entityService)) + .dataFetcher("deleteTag", new DeleteTagResolver(entityClient)) + .dataFetcher("updateChart", new MutableTypeResolver<>(chartType)) + .dataFetcher("updateDashboard", new MutableTypeResolver<>(dashboardType)) + .dataFetcher("updateNotebook", new MutableTypeResolver<>(notebookType)) + .dataFetcher("updateDataJob", new MutableTypeResolver<>(dataJobType)) + .dataFetcher("updateDataFlow", new MutableTypeResolver<>(dataFlowType)) + .dataFetcher("updateCorpUserProperties", new MutableTypeResolver<>(corpUserType)) + .dataFetcher("updateCorpGroupProperties", new MutableTypeResolver<>(corpGroupType)) + .dataFetcher( + "updateERModelRelationship", + new UpdateERModelRelationshipResolver(this.entityClient)) + .dataFetcher( + "createERModelRelationship", + new CreateERModelRelationshipResolver( + this.entityClient, this.erModelRelationshipService)) + .dataFetcher("addTag", new AddTagResolver(entityService)) + .dataFetcher("addTags", new AddTagsResolver(entityService)) + .dataFetcher("batchAddTags", new BatchAddTagsResolver(entityService)) + .dataFetcher("removeTag", new RemoveTagResolver(entityService)) + .dataFetcher("batchRemoveTags", new BatchRemoveTagsResolver(entityService)) + .dataFetcher("addTerm", new AddTermResolver(entityService)) + .dataFetcher("batchAddTerms", new BatchAddTermsResolver(entityService)) + .dataFetcher("addTerms", new AddTermsResolver(entityService)) + .dataFetcher("removeTerm", new RemoveTermResolver(entityService)) + .dataFetcher("batchRemoveTerms", new BatchRemoveTermsResolver(entityService)) + .dataFetcher("createPolicy", new UpsertPolicyResolver(this.entityClient)) + .dataFetcher("updatePolicy", new UpsertPolicyResolver(this.entityClient)) + .dataFetcher("deletePolicy", new DeletePolicyResolver(this.entityClient)) + .dataFetcher( + "updateDescription", + new UpdateDescriptionResolver(entityService, this.entityClient)) + .dataFetcher("addOwner", new AddOwnerResolver(entityService)) + .dataFetcher("addOwners", new AddOwnersResolver(entityService)) + .dataFetcher("batchAddOwners", new BatchAddOwnersResolver(entityService)) + .dataFetcher("removeOwner", new RemoveOwnerResolver(entityService)) + .dataFetcher("batchRemoveOwners", new BatchRemoveOwnersResolver(entityService)) + .dataFetcher("addLink", new AddLinkResolver(entityService, this.entityClient)) + .dataFetcher("removeLink", new RemoveLinkResolver(entityService)) + .dataFetcher("addGroupMembers", new AddGroupMembersResolver(this.groupService)) + .dataFetcher("removeGroupMembers", new RemoveGroupMembersResolver(this.groupService)) + .dataFetcher("createGroup", new CreateGroupResolver(this.groupService)) + .dataFetcher("removeUser", new RemoveUserResolver(this.entityClient)) + .dataFetcher("removeGroup", new RemoveGroupResolver(this.entityClient)) + .dataFetcher("updateUserStatus", new UpdateUserStatusResolver(this.entityClient)) + .dataFetcher( + "createDomain", new CreateDomainResolver(this.entityClient, this.entityService)) + .dataFetcher( + "moveDomain", new MoveDomainResolver(this.entityService, this.entityClient)) + .dataFetcher("deleteDomain", new DeleteDomainResolver(entityClient)) + .dataFetcher( + "setDomain", new SetDomainResolver(this.entityClient, this.entityService)) + .dataFetcher("batchSetDomain", new BatchSetDomainResolver(this.entityService)) + .dataFetcher( + "updateDeprecation", + new UpdateDeprecationResolver(this.entityClient, this.entityService)) + .dataFetcher( + "batchUpdateDeprecation", new BatchUpdateDeprecationResolver(entityService)) + .dataFetcher( + "unsetDomain", new UnsetDomainResolver(this.entityClient, this.entityService)) + .dataFetcher( + "createSecret", new CreateSecretResolver(this.entityClient, this.secretService)) + .dataFetcher("deleteSecret", new DeleteSecretResolver(this.entityClient)) + .dataFetcher( + "updateSecret", new UpdateSecretResolver(this.entityClient, this.secretService)) + .dataFetcher( + "createAccessToken", new CreateAccessTokenResolver(this.statefulTokenService)) + .dataFetcher( + "revokeAccessToken", + new RevokeAccessTokenResolver(this.entityClient, this.statefulTokenService)) + .dataFetcher( + "createIngestionSource", new UpsertIngestionSourceResolver(this.entityClient)) + .dataFetcher( + "updateIngestionSource", new UpsertIngestionSourceResolver(this.entityClient)) + .dataFetcher( + "deleteIngestionSource", new DeleteIngestionSourceResolver(this.entityClient)) + .dataFetcher( + "createIngestionExecutionRequest", + new CreateIngestionExecutionRequestResolver( + this.entityClient, this.ingestionConfiguration)) + .dataFetcher( + "cancelIngestionExecutionRequest", + new CancelIngestionExecutionRequestResolver(this.entityClient)) + .dataFetcher( + "createTestConnectionRequest", + new CreateTestConnectionRequestResolver( + this.entityClient, this.ingestionConfiguration)) + .dataFetcher( + "deleteAssertion", + new DeleteAssertionResolver(this.entityClient, this.entityService)) + .dataFetcher("createTest", new CreateTestResolver(this.entityClient)) + .dataFetcher("updateTest", new UpdateTestResolver(this.entityClient)) + .dataFetcher("deleteTest", new DeleteTestResolver(this.entityClient)) + .dataFetcher("reportOperation", new ReportOperationResolver(this.entityClient)) + .dataFetcher( + "createGlossaryTerm", + new CreateGlossaryTermResolver(this.entityClient, this.entityService)) + .dataFetcher( + "createGlossaryNode", + new CreateGlossaryNodeResolver(this.entityClient, this.entityService)) + .dataFetcher( + "updateParentNode", + new UpdateParentNodeResolver(this.entityService, this.entityClient)) + .dataFetcher( + "deleteGlossaryEntity", + new DeleteGlossaryEntityResolver(this.entityClient, this.entityService)) + .dataFetcher( + "updateName", new UpdateNameResolver(this.entityService, this.entityClient)) + .dataFetcher( + "addRelatedTerms", + new AddRelatedTermsResolver(this.entityService, this.entityClient)) + .dataFetcher( + "removeRelatedTerms", + new RemoveRelatedTermsResolver(this.entityService, this.entityClient)) + .dataFetcher( + "createNativeUserResetToken", + new CreateNativeUserResetTokenResolver(this.nativeUserService)) + .dataFetcher( + "batchUpdateSoftDeleted", new BatchUpdateSoftDeletedResolver(this.entityService)) + .dataFetcher("updateUserSetting", new UpdateUserSettingResolver(this.entityService)) + .dataFetcher("rollbackIngestion", new RollbackIngestionResolver(this.entityClient)) + .dataFetcher("batchAssignRole", new BatchAssignRoleResolver(this.roleService)) + .dataFetcher( + "createInviteToken", new CreateInviteTokenResolver(this.inviteTokenService)) + .dataFetcher( + "acceptRole", new AcceptRoleResolver(this.roleService, this.inviteTokenService)) + .dataFetcher("createPost", new CreatePostResolver(this.postService)) + .dataFetcher("deletePost", new DeletePostResolver(this.postService)) + .dataFetcher("updatePost", new UpdatePostResolver(this.postService)) + .dataFetcher( + "batchUpdateStepStates", new BatchUpdateStepStatesResolver(this.entityClient)) + .dataFetcher("createView", new CreateViewResolver(this.viewService)) + .dataFetcher("updateView", new UpdateViewResolver(this.viewService)) + .dataFetcher("deleteView", new DeleteViewResolver(this.viewService)) + .dataFetcher( + "updateGlobalViewsSettings", + new UpdateGlobalViewsSettingsResolver(this.settingsService)) + .dataFetcher( + "updateCorpUserViewsSettings", + new UpdateCorpUserViewsSettingsResolver(this.settingsService)) + .dataFetcher( + "updateLineage", + new UpdateLineageResolver(this.entityService, this.lineageService)) + .dataFetcher("updateEmbed", new UpdateEmbedResolver(this.entityService)) + .dataFetcher("createQuery", new CreateQueryResolver(this.queryService)) + .dataFetcher("updateQuery", new UpdateQueryResolver(this.queryService)) + .dataFetcher("deleteQuery", new DeleteQueryResolver(this.queryService)) + .dataFetcher( + "createDataProduct", new CreateDataProductResolver(this.dataProductService)) + .dataFetcher( + "updateDataProduct", new UpdateDataProductResolver(this.dataProductService)) + .dataFetcher( + "deleteDataProduct", new DeleteDataProductResolver(this.dataProductService)) + .dataFetcher( + "batchSetDataProduct", new BatchSetDataProductResolver(this.dataProductService)) + .dataFetcher( + "createOwnershipType", new CreateOwnershipTypeResolver(this.ownershipTypeService)) + .dataFetcher( + "updateOwnershipType", new UpdateOwnershipTypeResolver(this.ownershipTypeService)) + .dataFetcher( + "deleteOwnershipType", new DeleteOwnershipTypeResolver(this.ownershipTypeService)) + .dataFetcher("submitFormPrompt", new SubmitFormPromptResolver(this.formService)) + .dataFetcher("batchAssignForm", new BatchAssignFormResolver(this.formService)) + .dataFetcher( + "createDynamicFormAssignment", + new CreateDynamicFormAssignmentResolver(this.formService)) + .dataFetcher( + "verifyForm", new VerifyFormResolver(this.formService, this.groupService)) + .dataFetcher("batchRemoveForm", new BatchRemoveFormResolver(this.formService)) + .dataFetcher( + "upsertStructuredProperties", + new UpsertStructuredPropertiesResolver(this.entityClient)) + .dataFetcher("raiseIncident", new RaiseIncidentResolver(this.entityClient)) + .dataFetcher( + "updateIncidentStatus", + new UpdateIncidentStatusResolver(this.entityClient, this.entityService)); + if (featureFlags.isBusinessAttributeEntityEnabled()) { typeWiring - .dataFetcher("updateDataset", new MutableTypeResolver<>(datasetType)) - .dataFetcher("updateDatasets", new MutableTypeBatchResolver<>(datasetType)) - .dataFetcher( - "createTag", new CreateTagResolver(this.entityClient, this.entityService)) - .dataFetcher("updateTag", new MutableTypeResolver<>(tagType)) - .dataFetcher("setTagColor", new SetTagColorResolver(entityClient, entityService)) - .dataFetcher("deleteTag", new DeleteTagResolver(entityClient)) - .dataFetcher("updateChart", new MutableTypeResolver<>(chartType)) - .dataFetcher("updateDashboard", new MutableTypeResolver<>(dashboardType)) - .dataFetcher("updateNotebook", new MutableTypeResolver<>(notebookType)) - .dataFetcher("updateDataJob", new MutableTypeResolver<>(dataJobType)) - .dataFetcher("updateDataFlow", new MutableTypeResolver<>(dataFlowType)) - .dataFetcher("updateCorpUserProperties", new MutableTypeResolver<>(corpUserType)) - .dataFetcher("updateCorpGroupProperties", new MutableTypeResolver<>(corpGroupType)) - .dataFetcher( - "updateERModelRelationship", - new UpdateERModelRelationshipResolver(this.entityClient)) - .dataFetcher( - "createERModelRelationship", - new CreateERModelRelationshipResolver( - this.entityClient, this.erModelRelationshipService)) - .dataFetcher("addTag", new AddTagResolver(entityService)) - .dataFetcher("addTags", new AddTagsResolver(entityService)) - .dataFetcher("batchAddTags", new BatchAddTagsResolver(entityService)) - .dataFetcher("removeTag", new RemoveTagResolver(entityService)) - .dataFetcher("batchRemoveTags", new BatchRemoveTagsResolver(entityService)) - .dataFetcher("addTerm", new AddTermResolver(entityService)) - .dataFetcher("batchAddTerms", new BatchAddTermsResolver(entityService)) - .dataFetcher("addTerms", new AddTermsResolver(entityService)) - .dataFetcher("removeTerm", new RemoveTermResolver(entityService)) - .dataFetcher("batchRemoveTerms", new BatchRemoveTermsResolver(entityService)) - .dataFetcher("createPolicy", new UpsertPolicyResolver(this.entityClient)) - .dataFetcher("updatePolicy", new UpsertPolicyResolver(this.entityClient)) - .dataFetcher("deletePolicy", new DeletePolicyResolver(this.entityClient)) - .dataFetcher( - "updateDescription", - new UpdateDescriptionResolver(entityService, this.entityClient)) - .dataFetcher("addOwner", new AddOwnerResolver(entityService)) - .dataFetcher("addOwners", new AddOwnersResolver(entityService)) - .dataFetcher("batchAddOwners", new BatchAddOwnersResolver(entityService)) - .dataFetcher("removeOwner", new RemoveOwnerResolver(entityService)) - .dataFetcher("batchRemoveOwners", new BatchRemoveOwnersResolver(entityService)) - .dataFetcher("addLink", new AddLinkResolver(entityService, this.entityClient)) - .dataFetcher("removeLink", new RemoveLinkResolver(entityService)) - .dataFetcher("addGroupMembers", new AddGroupMembersResolver(this.groupService)) - .dataFetcher( - "removeGroupMembers", new RemoveGroupMembersResolver(this.groupService)) - .dataFetcher("createGroup", new CreateGroupResolver(this.groupService)) - .dataFetcher("removeUser", new RemoveUserResolver(this.entityClient)) - .dataFetcher("removeGroup", new RemoveGroupResolver(this.entityClient)) - .dataFetcher("updateUserStatus", new UpdateUserStatusResolver(this.entityClient)) - .dataFetcher( - "createDomain", new CreateDomainResolver(this.entityClient, this.entityService)) - .dataFetcher( - "moveDomain", new MoveDomainResolver(this.entityService, this.entityClient)) - .dataFetcher("deleteDomain", new DeleteDomainResolver(entityClient)) - .dataFetcher( - "setDomain", new SetDomainResolver(this.entityClient, this.entityService)) - .dataFetcher("batchSetDomain", new BatchSetDomainResolver(this.entityService)) - .dataFetcher( - "updateDeprecation", - new UpdateDeprecationResolver(this.entityClient, this.entityService)) - .dataFetcher( - "batchUpdateDeprecation", new BatchUpdateDeprecationResolver(entityService)) - .dataFetcher( - "unsetDomain", new UnsetDomainResolver(this.entityClient, this.entityService)) - .dataFetcher( - "createSecret", new CreateSecretResolver(this.entityClient, this.secretService)) - .dataFetcher("deleteSecret", new DeleteSecretResolver(this.entityClient)) - .dataFetcher( - "updateSecret", new UpdateSecretResolver(this.entityClient, this.secretService)) - .dataFetcher( - "createAccessToken", new CreateAccessTokenResolver(this.statefulTokenService)) - .dataFetcher( - "revokeAccessToken", - new RevokeAccessTokenResolver(this.entityClient, this.statefulTokenService)) - .dataFetcher( - "createIngestionSource", new UpsertIngestionSourceResolver(this.entityClient)) - .dataFetcher( - "updateIngestionSource", new UpsertIngestionSourceResolver(this.entityClient)) - .dataFetcher( - "deleteIngestionSource", new DeleteIngestionSourceResolver(this.entityClient)) - .dataFetcher( - "createIngestionExecutionRequest", - new CreateIngestionExecutionRequestResolver( - this.entityClient, this.ingestionConfiguration)) - .dataFetcher( - "cancelIngestionExecutionRequest", - new CancelIngestionExecutionRequestResolver(this.entityClient)) - .dataFetcher( - "createTestConnectionRequest", - new CreateTestConnectionRequestResolver( - this.entityClient, this.ingestionConfiguration)) - .dataFetcher( - "deleteAssertion", - new DeleteAssertionResolver(this.entityClient, this.entityService)) - .dataFetcher("createTest", new CreateTestResolver(this.entityClient)) - .dataFetcher("updateTest", new UpdateTestResolver(this.entityClient)) - .dataFetcher("deleteTest", new DeleteTestResolver(this.entityClient)) - .dataFetcher("reportOperation", new ReportOperationResolver(this.entityClient)) - .dataFetcher( - "createGlossaryTerm", - new CreateGlossaryTermResolver(this.entityClient, this.entityService)) - .dataFetcher( - "createGlossaryNode", - new CreateGlossaryNodeResolver(this.entityClient, this.entityService)) - .dataFetcher( - "updateParentNode", - new UpdateParentNodeResolver(this.entityService, this.entityClient)) - .dataFetcher( - "deleteGlossaryEntity", - new DeleteGlossaryEntityResolver(this.entityClient, this.entityService)) - .dataFetcher( - "updateName", new UpdateNameResolver(this.entityService, this.entityClient)) - .dataFetcher( - "addRelatedTerms", - new AddRelatedTermsResolver(this.entityService, this.entityClient)) - .dataFetcher( - "removeRelatedTerms", - new RemoveRelatedTermsResolver(this.entityService, this.entityClient)) - .dataFetcher( - "createNativeUserResetToken", - new CreateNativeUserResetTokenResolver(this.nativeUserService)) - .dataFetcher( - "batchUpdateSoftDeleted", - new BatchUpdateSoftDeletedResolver(this.entityService)) - .dataFetcher("updateUserSetting", new UpdateUserSettingResolver(this.entityService)) - .dataFetcher("rollbackIngestion", new RollbackIngestionResolver(this.entityClient)) - .dataFetcher("batchAssignRole", new BatchAssignRoleResolver(this.roleService)) - .dataFetcher( - "createInviteToken", new CreateInviteTokenResolver(this.inviteTokenService)) - .dataFetcher( - "acceptRole", new AcceptRoleResolver(this.roleService, this.inviteTokenService)) - .dataFetcher("createPost", new CreatePostResolver(this.postService)) - .dataFetcher("deletePost", new DeletePostResolver(this.postService)) - .dataFetcher("updatePost", new UpdatePostResolver(this.postService)) .dataFetcher( - "batchUpdateStepStates", new BatchUpdateStepStatesResolver(this.entityClient)) - .dataFetcher("createView", new CreateViewResolver(this.viewService)) - .dataFetcher("updateView", new UpdateViewResolver(this.viewService)) - .dataFetcher("deleteView", new DeleteViewResolver(this.viewService)) + "createBusinessAttribute", + new CreateBusinessAttributeResolver( + this.entityClient, this.entityService, this.businessAttributeService)) .dataFetcher( - "updateGlobalViewsSettings", - new UpdateGlobalViewsSettingsResolver(this.settingsService)) + "updateBusinessAttribute", + new UpdateBusinessAttributeResolver( + this.entityClient, this.businessAttributeService)) .dataFetcher( - "updateCorpUserViewsSettings", - new UpdateCorpUserViewsSettingsResolver(this.settingsService)) + "deleteBusinessAttribute", + new DeleteBusinessAttributeResolver(this.entityClient)) .dataFetcher( - "updateLineage", - new UpdateLineageResolver(this.entityService, this.lineageService)) - .dataFetcher("updateEmbed", new UpdateEmbedResolver(this.entityService)) - .dataFetcher("createQuery", new CreateQueryResolver(this.queryService)) - .dataFetcher("updateQuery", new UpdateQueryResolver(this.queryService)) - .dataFetcher("deleteQuery", new DeleteQueryResolver(this.queryService)) + "addBusinessAttribute", new AddBusinessAttributeResolver(this.entityService)) .dataFetcher( - "createDataProduct", new CreateDataProductResolver(this.dataProductService)) - .dataFetcher( - "updateDataProduct", new UpdateDataProductResolver(this.dataProductService)) - .dataFetcher( - "deleteDataProduct", new DeleteDataProductResolver(this.dataProductService)) - .dataFetcher( - "batchSetDataProduct", new BatchSetDataProductResolver(this.dataProductService)) - .dataFetcher( - "createOwnershipType", - new CreateOwnershipTypeResolver(this.ownershipTypeService)) - .dataFetcher( - "updateOwnershipType", - new UpdateOwnershipTypeResolver(this.ownershipTypeService)) - .dataFetcher( - "deleteOwnershipType", - new DeleteOwnershipTypeResolver(this.ownershipTypeService)) - .dataFetcher("submitFormPrompt", new SubmitFormPromptResolver(this.formService)) - .dataFetcher("batchAssignForm", new BatchAssignFormResolver(this.formService)) - .dataFetcher( - "createDynamicFormAssignment", - new CreateDynamicFormAssignmentResolver(this.formService)) - .dataFetcher( - "verifyForm", new VerifyFormResolver(this.formService, this.groupService)) - .dataFetcher("batchRemoveForm", new BatchRemoveFormResolver(this.formService)) - .dataFetcher( - "upsertStructuredProperties", - new UpsertStructuredPropertiesResolver(this.entityClient)) - .dataFetcher("raiseIncident", new RaiseIncidentResolver(this.entityClient)) - .dataFetcher( - "updateIncidentStatus", - new UpdateIncidentStatusResolver(this.entityClient, this.entityService))); + "removeBusinessAttribute", + new RemoveBusinessAttributeResolver(this.entityService)); + } + return typeWiring; + }); } private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder) { @@ -2934,4 +2973,40 @@ private void configureRoleResolvers(final RuntimeWiring.Builder builder) { "Role", typeWiring -> typeWiring.dataFetcher("isAssignedToMe", new IsAssignedToMeResolver())); } + + private void configureBusinessAttributeResolver(final RuntimeWiring.Builder builder) { + builder + .type( + "BusinessAttribute", + typeWiring -> + typeWiring + .dataFetcher("exists", new EntityExistsResolver(entityService)) + .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))) + .type( + "ListBusinessAttributesResult", + typeWiring -> + typeWiring.dataFetcher( + "businessAttributes", + new LoadableTypeBatchResolver<>( + businessAttributeType, + (env) -> + ((ListBusinessAttributesResult) env.getSource()) + .getBusinessAttributes().stream() + .map(BusinessAttribute::getUrn) + .collect(Collectors.toList())))); + } + + private void configureBusinessAttributeAssociationResolver(final RuntimeWiring.Builder builder) { + builder.type( + "BusinessAttributeAssociation", + typeWiring -> + typeWiring.dataFetcher( + "businessAttribute", + new LoadableTypeResolver<>( + businessAttributeType, + (env) -> + ((BusinessAttributeAssociation) env.getSource()) + .getBusinessAttribute() + .getUrn()))); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java index 5f5e1c929f6ac7..abb491814c278e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java @@ -23,6 +23,7 @@ import com.linkedin.metadata.graph.SiblingGraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.recommendation.RecommendationsService; +import com.linkedin.metadata.service.BusinessAttributeService; import com.linkedin.metadata.service.DataProductService; import com.linkedin.metadata.service.ERModelRelationshipService; import com.linkedin.metadata.service.FormService; @@ -81,6 +82,7 @@ public class GmsGraphQLEngineArgs { RestrictedService restrictedService; int graphQLQueryComplexityLimit; int graphQLQueryDepthLimit; + BusinessAttributeService businessAttributeService; // any fork specific args should go below this line } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java index 8bc716f4ff4db5..2a4f75cf6055a0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java @@ -19,4 +19,5 @@ public class FeatureFlags { private boolean showAccessManagement = false; private boolean nestedDomainsEnabled = false; private boolean schemaFieldEntityFetchEnabled = false; + private boolean businessAttributeEntityEnabled = false; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java index 30817d1c621529..d1b4cc3f715ead 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java @@ -15,6 +15,7 @@ import com.linkedin.datahub.graphql.generated.AuthenticatedUser; import com.linkedin.datahub.graphql.generated.CorpUser; import com.linkedin.datahub.graphql.generated.PlatformPrivileges; +import com.linkedin.datahub.graphql.resolvers.businessattribute.BusinessAttributeAuthorizationUtils; import com.linkedin.datahub.graphql.types.corpuser.mappers.CorpUserMapper; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.client.EntityClient; @@ -86,7 +87,10 @@ public CompletableFuture get(DataFetchingEnvironment environm AuthorizationUtils.canManageOwnershipTypes(context)); platformPrivileges.setManageGlobalAnnouncements( AuthorizationUtils.canManageGlobalAnnouncements(context)); - + platformPrivileges.setCreateBusinessAttributes( + BusinessAttributeAuthorizationUtils.canCreateBusinessAttribute(context)); + platformPrivileges.setManageBusinessAttributes( + BusinessAttributeAuthorizationUtils.canManageBusinessAttribute(context)); // Construct and return authenticated user object. final AuthenticatedUser authUser = new AuthenticatedUser(); authUser.setCorpUser(corpUser); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/AddBusinessAttributeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/AddBusinessAttributeResolver.java new file mode 100644 index 00000000000000..eb477dff088abe --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/AddBusinessAttributeResolver.java @@ -0,0 +1,109 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.buildMetadataChangeProposalWithUrn; +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_ASPECT; + +import com.linkedin.businessattribute.BusinessAttributeAssociation; +import com.linkedin.businessattribute.BusinessAttributes; +import com.linkedin.common.urn.BusinessAttributeUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AddBusinessAttributeInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class AddBusinessAttributeResolver implements DataFetcher> { + private final EntityService entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final AddBusinessAttributeInput input = + bindArgument(environment.getArgument("input"), AddBusinessAttributeInput.class); + final Urn businessAttributeUrn = UrnUtils.getUrn(input.getBusinessAttributeUrn()); + final List resourceRefInputs = input.getResourceUrn(); + validateBusinessAttribute(businessAttributeUrn); + return CompletableFuture.supplyAsync( + () -> { + try { + addBusinessAttributeToResource( + businessAttributeUrn, + resourceRefInputs, + UrnUtils.getUrn(context.getActorUrn()), + entityService); + return true; + } catch (Exception e) { + log.error( + String.format( + "Failed to add Business Attribute %s to resources %s", + businessAttributeUrn, resourceRefInputs)); + throw new RuntimeException( + String.format( + "Failed to add Business Attribute %s to resources %s", + businessAttributeUrn, resourceRefInputs), + e); + } + }); + } + + private void validateBusinessAttribute(Urn businessAttributeUrn) { + if (!entityService.exists(businessAttributeUrn, true)) { + throw new IllegalArgumentException( + String.format("This urn does not exist: %s", businessAttributeUrn)); + } + } + + private void addBusinessAttributeToResource( + Urn businessAttributeUrn, + List resourceRefInputs, + Urn actorUrn, + EntityService entityService) + throws URISyntaxException { + List proposals = new ArrayList<>(); + for (ResourceRefInput resourceRefInput : resourceRefInputs) { + proposals.add( + buildAddBusinessAttributeToEntityProposal( + businessAttributeUrn, resourceRefInput, entityService, actorUrn)); + } + EntityUtils.ingestChangeProposals(proposals, entityService, actorUrn, false); + } + + private MetadataChangeProposal buildAddBusinessAttributeToEntityProposal( + Urn businessAttributeUrn, + ResourceRefInput resource, + EntityService entityService, + Urn actorUrn) + throws URISyntaxException { + BusinessAttributes businessAttributes = + (BusinessAttributes) + EntityUtils.getAspectFromEntity( + resource.getResourceUrn(), + BUSINESS_ATTRIBUTE_ASPECT, + entityService, + new BusinessAttributes()); + if (!businessAttributes.hasBusinessAttribute()) { + businessAttributes.setBusinessAttribute(new BusinessAttributeAssociation()); + } + BusinessAttributeAssociation businessAttributeAssociation = + businessAttributes.getBusinessAttribute(); + businessAttributeAssociation.setBusinessAttributeUrn( + BusinessAttributeUrn.createFromUrn(businessAttributeUrn)); + businessAttributes.setBusinessAttribute(businessAttributeAssociation); + return buildMetadataChangeProposalWithUrn( + UrnUtils.getUrn(resource.getResourceUrn()), BUSINESS_ATTRIBUTE_ASPECT, businessAttributes); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/BusinessAttributeAuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/BusinessAttributeAuthorizationUtils.java new file mode 100644 index 00000000000000..041f5e9ade77f0 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/BusinessAttributeAuthorizationUtils.java @@ -0,0 +1,37 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import com.datahub.authorization.AuthUtil; +import com.datahub.authorization.ConjunctivePrivilegeGroup; +import com.datahub.authorization.DisjunctivePrivilegeGroup; +import com.google.common.collect.ImmutableList; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.metadata.authorization.PoliciesConfig; +import javax.annotation.Nonnull; + +public class BusinessAttributeAuthorizationUtils { + private BusinessAttributeAuthorizationUtils() {} + + public static boolean canCreateBusinessAttribute(@Nonnull QueryContext context) { + final DisjunctivePrivilegeGroup orPrivilegeGroups = + new DisjunctivePrivilegeGroup( + ImmutableList.of( + new ConjunctivePrivilegeGroup( + ImmutableList.of(PoliciesConfig.CREATE_BUSINESS_ATTRIBUTE_PRIVILEGE.getType())), + new ConjunctivePrivilegeGroup( + ImmutableList.of( + PoliciesConfig.MANAGE_BUSINESS_ATTRIBUTE_PRIVILEGE.getType())))); + return AuthUtil.isAuthorized( + context.getAuthorizer(), context.getActorUrn(), orPrivilegeGroups, null); + } + + public static boolean canManageBusinessAttribute(@Nonnull QueryContext context) { + final DisjunctivePrivilegeGroup orPrivilegeGroups = + new DisjunctivePrivilegeGroup( + ImmutableList.of( + new ConjunctivePrivilegeGroup( + ImmutableList.of( + PoliciesConfig.MANAGE_BUSINESS_ATTRIBUTE_PRIVILEGE.getType())))); + return AuthUtil.isAuthorized( + context.getAuthorizer(), context.getActorUrn(), orPrivilegeGroups, null); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeResolver.java new file mode 100644 index 00000000000000..3c4f6315016fd2 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeResolver.java @@ -0,0 +1,130 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.buildMetadataChangeProposalWithKey; +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME; +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME; + +import com.linkedin.businessattribute.BusinessAttributeInfo; +import com.linkedin.businessattribute.BusinessAttributeKey; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; +import com.linkedin.datahub.graphql.generated.BusinessAttribute; +import com.linkedin.datahub.graphql.generated.CreateBusinessAttributeInput; +import com.linkedin.datahub.graphql.generated.OwnerEntityType; +import com.linkedin.datahub.graphql.resolvers.mutate.util.BusinessAttributeUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils; +import com.linkedin.datahub.graphql.types.businessattribute.mappers.BusinessAttributeMapper; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.service.BusinessAttributeService; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class CreateBusinessAttributeResolver + implements DataFetcher> { + private final EntityClient _entityClient; + private final EntityService _entityService; + private final BusinessAttributeService businessAttributeService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + CreateBusinessAttributeInput input = + bindArgument(environment.getArgument("input"), CreateBusinessAttributeInput.class); + if (!BusinessAttributeAuthorizationUtils.canCreateBusinessAttribute(context)) { + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + } + return CompletableFuture.supplyAsync( + () -> { + try { + final BusinessAttributeKey businessAttributeKey = new BusinessAttributeKey(); + String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); + businessAttributeKey.setId(id); + + if (_entityClient.exists( + EntityKeyUtils.convertEntityKeyToUrn( + businessAttributeKey, BUSINESS_ATTRIBUTE_ENTITY_NAME), + context.getAuthentication())) { + throw new IllegalArgumentException("This Business Attribute already exists!"); + } + + if (BusinessAttributeUtils.hasNameConflict(input.getName(), context, _entityClient)) { + throw new DataHubGraphQLException( + String.format( + "\"%s\" already exists as Business Attribute. Please pick a unique name.", + input.getName()), + DataHubGraphQLErrorCode.CONFLICT); + } + + // Create the MCP + final MetadataChangeProposal changeProposal = + buildMetadataChangeProposalWithKey( + businessAttributeKey, + BUSINESS_ATTRIBUTE_ENTITY_NAME, + BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + mapBusinessAttributeInfo(input, context)); + + // Ingest the MCP + Urn businessAttributeUrn = + UrnUtils.getUrn( + _entityClient.ingestProposal(changeProposal, context.getAuthentication())); + OwnerUtils.addCreatorAsOwner( + context, + businessAttributeUrn.toString(), + OwnerEntityType.CORP_USER, + _entityService); + return BusinessAttributeMapper.map( + context, + businessAttributeService.getBusinessAttributeEntityResponse( + businessAttributeUrn, context.getAuthentication())); + + } catch (DataHubGraphQLException e) { + throw e; + } catch (Exception e) { + log.error( + "Failed to create Business Attribute with name: {}: {}", + input.getName(), + e.getMessage()); + throw new RuntimeException( + String.format("Failed to create Business Attribute with name: %s", input.getName()), + e); + } + }); + } + + private BusinessAttributeInfo mapBusinessAttributeInfo( + CreateBusinessAttributeInput input, QueryContext context) { + final BusinessAttributeInfo info = new BusinessAttributeInfo(); + info.setFieldPath(input.getName(), SetMode.DISALLOW_NULL); + info.setName(input.getName(), SetMode.DISALLOW_NULL); + info.setDescription(input.getDescription(), SetMode.IGNORE_NULL); + info.setType( + BusinessAttributeUtils.mapSchemaFieldDataType(input.getType()), SetMode.IGNORE_NULL); + info.setCreated( + new AuditStamp() + .setActor(UrnUtils.getUrn(context.getActorUrn())) + .setTime(System.currentTimeMillis())); + info.setLastModified( + new AuditStamp() + .setActor(UrnUtils.getUrn(context.getActorUrn())) + .setTime(System.currentTimeMillis())); + return info; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/DeleteBusinessAttributeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/DeleteBusinessAttributeResolver.java new file mode 100644 index 00000000000000..b397c27834392b --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/DeleteBusinessAttributeResolver.java @@ -0,0 +1,58 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.entity.client.EntityClient; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** Resolver responsible for hard deleting a particular Business Attribute */ +@Slf4j +@RequiredArgsConstructor +public class DeleteBusinessAttributeResolver implements DataFetcher> { + private final EntityClient _entityClient; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final Urn businessAttributeUrn = UrnUtils.getUrn(environment.getArgument("urn")); + if (!BusinessAttributeAuthorizationUtils.canManageBusinessAttribute(context)) { + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + } + if (!_entityClient.exists(businessAttributeUrn, context.getAuthentication())) { + throw new RuntimeException( + String.format("This urn does not exist: %s", businessAttributeUrn)); + } + return CompletableFuture.supplyAsync( + () -> { + try { + _entityClient.deleteEntity(businessAttributeUrn, context.getAuthentication()); + CompletableFuture.runAsync( + () -> { + try { + _entityClient.deleteEntityReferences( + businessAttributeUrn, context.getAuthentication()); + } catch (Exception e) { + log.error( + String.format( + "Exception while attempting to clear all entity references for Business Attribute with urn %s", + businessAttributeUrn), + e); + } + }); + return true; + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Failed to delete Business Attribute with urn %s", businessAttributeUrn), + e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/ListBusinessAttributesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/ListBusinessAttributesResolver.java new file mode 100644 index 00000000000000..00ea5975d260e1 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/ListBusinessAttributesResolver.java @@ -0,0 +1,92 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BusinessAttribute; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.ListBusinessAttributesInput; +import com.linkedin.datahub.graphql.generated.ListBusinessAttributesResult; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchResult; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +/** Resolver used for listing Business Attributes. */ +@Slf4j +public class ListBusinessAttributesResolver + implements DataFetcher> { + + private static final Integer DEFAULT_START = 0; + private static final Integer DEFAULT_COUNT = 20; + private static final String DEFAULT_QUERY = ""; + + private final EntityClient _entityClient; + + public ListBusinessAttributesResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get( + final DataFetchingEnvironment environment) throws Exception { + + final QueryContext context = environment.getContext(); + final ListBusinessAttributesInput input = + bindArgument(environment.getArgument("input"), ListBusinessAttributesInput.class); + + return CompletableFuture.supplyAsync( + () -> { + final Integer start = input.getStart() == null ? DEFAULT_START : input.getStart(); + final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount(); + final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery(); + + try { + + final SearchResult gmsResult = + _entityClient.search( + context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)), + Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME, + query, + Collections.emptyMap(), + start, + count); + + final ListBusinessAttributesResult result = new ListBusinessAttributesResult(); + result.setStart(gmsResult.getFrom()); + result.setCount(gmsResult.getPageSize()); + result.setTotal(gmsResult.getNumEntities()); + result.setBusinessAttributes( + mapUnresolvedBusinessAttributes( + gmsResult.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()))); + return result; + } catch (Exception e) { + throw new RuntimeException("Failed to list Business Attributes", e); + } + }); + } + + private List mapUnresolvedBusinessAttributes(final List entityUrns) { + final List results = new ArrayList<>(); + for (final Urn urn : entityUrns) { + final BusinessAttribute unresolvedBusinessAttribute = new BusinessAttribute(); + unresolvedBusinessAttribute.setUrn(urn.toString()); + unresolvedBusinessAttribute.setType(EntityType.BUSINESS_ATTRIBUTE); + results.add(unresolvedBusinessAttribute); + } + return results; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/RemoveBusinessAttributeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/RemoveBusinessAttributeResolver.java new file mode 100644 index 00000000000000..63e9ac562a6586 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/RemoveBusinessAttributeResolver.java @@ -0,0 +1,82 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.buildMetadataChangeProposalWithUrn; +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_ASPECT; + +import com.linkedin.businessattribute.BusinessAttributes; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AddBusinessAttributeInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class RemoveBusinessAttributeResolver implements DataFetcher> { + private final EntityService entityService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final AddBusinessAttributeInput input = + bindArgument(environment.getArgument("input"), AddBusinessAttributeInput.class); + final Urn businessAttributeUrn = UrnUtils.getUrn(input.getBusinessAttributeUrn()); + final List resourceRefInputs = input.getResourceUrn(); + + return CompletableFuture.supplyAsync( + () -> { + try { + removeBusinessAttribute(resourceRefInputs, UrnUtils.getUrn(context.getActorUrn())); + return true; + } catch (Exception e) { + log.error( + String.format( + "Failed to remove Business Attribute with urn %s from resources %s", + businessAttributeUrn, resourceRefInputs)); + throw new RuntimeException( + String.format( + "Failed to remove Business Attribute with urn %s from resources %s", + businessAttributeUrn, resourceRefInputs), + e); + } + }); + } + + private void removeBusinessAttribute(List resourceRefInputs, Urn actorUrn) { + List proposals = new ArrayList<>(); + for (ResourceRefInput resourceRefInput : resourceRefInputs) { + proposals.add( + buildRemoveBusinessAttributeFromResourceProposal(resourceRefInput, entityService)); + } + EntityUtils.ingestChangeProposals(proposals, entityService, actorUrn, false); + } + + private MetadataChangeProposal buildRemoveBusinessAttributeFromResourceProposal( + ResourceRefInput resource, EntityService entityService) { + BusinessAttributes businessAttributes = + (BusinessAttributes) + EntityUtils.getAspectFromEntity( + resource.getResourceUrn(), + BUSINESS_ATTRIBUTE_ASPECT, + entityService, + new BusinessAttributes()); + if (!businessAttributes.hasBusinessAttribute()) { + throw new RuntimeException( + String.format("Schema field has not attached with business attribute")); + } + businessAttributes.removeBusinessAttribute(); + return buildMetadataChangeProposalWithUrn( + UrnUtils.getUrn(resource.getResourceUrn()), BUSINESS_ATTRIBUTE_ASPECT, businessAttributes); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/UpdateBusinessAttributeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/UpdateBusinessAttributeResolver.java new file mode 100644 index 00000000000000..32724ba13e8ac4 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/businessattribute/UpdateBusinessAttributeResolver.java @@ -0,0 +1,147 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.datahub.authentication.Authentication; +import com.linkedin.businessattribute.BusinessAttributeInfo; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; +import com.linkedin.datahub.graphql.generated.BusinessAttribute; +import com.linkedin.datahub.graphql.generated.UpdateBusinessAttributeInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.BusinessAttributeUtils; +import com.linkedin.datahub.graphql.types.businessattribute.mappers.BusinessAttributeMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.AspectUtils; +import com.linkedin.metadata.service.BusinessAttributeService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class UpdateBusinessAttributeResolver + implements DataFetcher> { + + private final EntityClient _entityClient; + private final BusinessAttributeService businessAttributeService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) + throws Exception { + QueryContext context = environment.getContext(); + UpdateBusinessAttributeInput input = + bindArgument(environment.getArgument("input"), UpdateBusinessAttributeInput.class); + final Urn businessAttributeUrn = UrnUtils.getUrn(environment.getArgument("urn")); + if (!BusinessAttributeAuthorizationUtils.canCreateBusinessAttribute(context)) { + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + } + if (!_entityClient.exists(businessAttributeUrn, context.getAuthentication())) { + throw new RuntimeException( + String.format("This urn does not exist: %s", businessAttributeUrn)); + } + return CompletableFuture.supplyAsync( + () -> { + try { + Urn updatedBusinessAttributeUrn = + updateBusinessAttribute(input, businessAttributeUrn, context); + return BusinessAttributeMapper.map( + context, + businessAttributeService.getBusinessAttributeEntityResponse( + updatedBusinessAttributeUrn, context.getAuthentication())); + } catch (DataHubGraphQLException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Failed to update Business Attribute with urn %s", businessAttributeUrn), + e); + } + }); + } + + private Urn updateBusinessAttribute( + UpdateBusinessAttributeInput input, Urn businessAttributeUrn, QueryContext context) { + try { + BusinessAttributeInfo businessAttributeInfo = + getBusinessAttributeInfo(businessAttributeUrn, context.getAuthentication()); + // 1. Check whether the Business Attribute exists + if (businessAttributeInfo == null) { + throw new IllegalArgumentException( + String.format( + "Failed to update Business Attribute. Business Attribute with urn %s does not exist.", + businessAttributeUrn)); + } + + // 2. Apply changes to existing Business Attribute + if (Objects.nonNull(input.getName())) { + if (BusinessAttributeUtils.hasNameConflict(input.getName(), context, _entityClient)) { + throw new DataHubGraphQLException( + String.format( + "\"%s\" already exists as Business Attribute. Please pick a unique name.", + input.getName()), + DataHubGraphQLErrorCode.CONFLICT); + } + businessAttributeInfo.setName(input.getName()); + businessAttributeInfo.setFieldPath(input.getName()); + } + if (Objects.nonNull(input.getDescription())) { + businessAttributeInfo.setDescription(input.getDescription()); + } + if (Objects.nonNull(input.getType())) { + businessAttributeInfo.setType( + BusinessAttributeUtils.mapSchemaFieldDataType(input.getType())); + } + businessAttributeInfo.setLastModified( + new AuditStamp() + .setActor(UrnUtils.getUrn(context.getActorUrn())) + .setTime(System.currentTimeMillis())); + // 3. Write changes to GMS + return UrnUtils.getUrn( + _entityClient.ingestProposal( + AspectUtils.buildMetadataChangeProposal( + businessAttributeUrn, + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + businessAttributeInfo), + context.getAuthentication())); + + } catch (DataHubGraphQLException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Nullable + private BusinessAttributeInfo getBusinessAttributeInfo( + @Nonnull final Urn businessAttributeUrn, @Nonnull final Authentication authentication) { + Objects.requireNonNull(businessAttributeUrn, "businessAttributeUrn must not be null"); + Objects.requireNonNull(authentication, "authentication must not be null"); + final EntityResponse response = + businessAttributeService.getBusinessAttributeEntityResponse( + businessAttributeUrn, authentication); + if (response != null + && response.getAspects().containsKey(Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME)) { + return new BusinessAttributeInfo( + response + .getAspects() + .get(Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME) + .getValue() + .data()); + } + // No aspect found + return null; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index d884afb36a280a..c05009e146308e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -176,6 +176,7 @@ public CompletableFuture get(final DataFetchingEnvironment environmen final FeatureFlagsConfig featureFlagsConfig = FeatureFlagsConfig.builder() .setShowSearchFiltersV2(_featureFlags.isShowSearchFiltersV2()) + .setBusinessAttributeEntityEnabled(_featureFlags.isBusinessAttributeEntityEnabled()) .setReadOnlyModeEnabled(_featureFlags.isReadOnlyModeEnabled()) .setShowBrowseV2(_featureFlags.isShowBrowseV2()) .setShowAcrylInfo(_featureFlags.isShowAcrylInfo()) @@ -268,6 +269,10 @@ private EntityType mapResourceTypeToEntityType(final String resourceType) { .getResourceType() .equals(resourceType)) { return EntityType.ER_MODEL_RELATIONSHIP; + } else if (com.linkedin.metadata.authorization.PoliciesConfig.BUSINESS_ATTRIBUTE_PRIVILEGES + .getResourceType() + .equals(resourceType)) { + return EntityType.BUSINESS_ATTRIBUTE; } else { return null; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/DescriptionUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/DescriptionUtils.java index ab151d6244f489..5f1ffb6a94b991 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/DescriptionUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/DescriptionUtils.java @@ -5,6 +5,7 @@ import com.datahub.authorization.ConjunctivePrivilegeGroup; import com.datahub.authorization.DisjunctivePrivilegeGroup; import com.google.common.collect.ImmutableList; +import com.linkedin.businessattribute.BusinessAttributeInfo; import com.linkedin.common.urn.Urn; import com.linkedin.container.EditableContainerProperties; import com.linkedin.datahub.graphql.QueryContext; @@ -453,4 +454,24 @@ public static void updateDataProductDescription( actor, entityService); } + + public static void updateBusinessAttributeDescription( + String newDescription, Urn resourceUrn, Urn actor, EntityService entityService) { + BusinessAttributeInfo businessAttributeInfo = + (BusinessAttributeInfo) + EntityUtils.getAspectFromEntity( + resourceUrn.toString(), + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + entityService, + new BusinessAttributeInfo()); + if (businessAttributeInfo != null) { + businessAttributeInfo.setDescription(newDescription); + } + persistAspect( + resourceUrn, + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + businessAttributeInfo, + actor, + entityService); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateDescriptionResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateDescriptionResolver.java index 13a8427633caee..d1cf5ed9feb2ae 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateDescriptionResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateDescriptionResolver.java @@ -63,6 +63,8 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw return updateMlPrimaryKeyDescription(targetUrn, input, environment.getContext()); case Constants.DATA_PRODUCT_ENTITY_NAME: return updateDataProductDescription(targetUrn, input, environment.getContext()); + case Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME: + return updateBusinessAttributeDescription(targetUrn, input, environment.getContext()); default: throw new RuntimeException( String.format( @@ -444,4 +446,31 @@ private CompletableFuture updateDataProductDescription( } }); } + + private CompletableFuture updateBusinessAttributeDescription( + Urn targetUrn, DescriptionUpdateInput input, QueryContext context) { + return CompletableFuture.supplyAsync( + () -> { + // check if user has the rights to update description for business attribute + if (!DescriptionUtils.isAuthorizedToUpdateDescription(context, targetUrn)) { + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + } + + // validate label input + DescriptionUtils.validateLabelInput(targetUrn, _entityService); + + try { + Urn actor = CorpuserUrn.createFromString(context.getActorUrn()); + DescriptionUtils.updateBusinessAttributeDescription( + input.getDescription(), targetUrn, actor, _entityService); + return true; + } catch (Exception e) { + log.error( + "Failed to perform update against input {}, {}", input.toString(), e.getMessage()); + throw new RuntimeException( + String.format("Failed to perform update against input %s", input.toString()), e); + } + }); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java index 8e4a96637e04dc..e501ac7ae87e7a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java @@ -3,6 +3,7 @@ import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.persistAspect; +import com.linkedin.businessattribute.BusinessAttributeInfo; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; @@ -12,7 +13,9 @@ import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; import com.linkedin.datahub.graphql.generated.UpdateNameInput; +import com.linkedin.datahub.graphql.resolvers.businessattribute.BusinessAttributeAuthorizationUtils; import com.linkedin.datahub.graphql.resolvers.dataproduct.DataProductAuthorizationUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.BusinessAttributeUtils; import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.datahub.graphql.resolvers.mutate.util.GlossaryUtils; import com.linkedin.dataproduct.DataProductProperties; @@ -63,6 +66,8 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw return updateGroupName(targetUrn, input, environment.getContext()); case Constants.DATA_PRODUCT_ENTITY_NAME: return updateDataProductName(targetUrn, input, environment.getContext()); + case Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME: + return updateBusinessAttributeName(targetUrn, input, environment.getContext()); default: throw new RuntimeException( String.format( @@ -268,4 +273,48 @@ private Boolean updateDataProductName( String.format("Failed to perform update against input %s", input), e); } } + + private Boolean updateBusinessAttributeName( + Urn targetUrn, UpdateNameInput input, QueryContext context) { + if (!BusinessAttributeAuthorizationUtils.canManageBusinessAttribute(context)) { + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + } + try { + BusinessAttributeInfo businessAttributeInfo = + (BusinessAttributeInfo) + EntityUtils.getAspectFromEntity( + targetUrn.toString(), + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + _entityService, + null); + if (businessAttributeInfo == null) { + throw new IllegalArgumentException("Business Attribute does not exist"); + } + + if (BusinessAttributeUtils.hasNameConflict(input.getName(), context, _entityClient)) { + throw new DataHubGraphQLException( + String.format( + "\"%s\" already exists as Business Attribute. Please pick a unique name.", + input.getName()), + DataHubGraphQLErrorCode.CONFLICT); + } + + businessAttributeInfo.setFieldPath(input.getName()); + businessAttributeInfo.setName(input.getName()); + Urn actor = CorpuserUrn.createFromString(context.getActorUrn()); + persistAspect( + targetUrn, + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + businessAttributeInfo, + actor, + _entityService); + return true; + } catch (DataHubGraphQLException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/BusinessAttributeUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/BusinessAttributeUtils.java new file mode 100644 index 00000000000000..25dc36f74ef73a --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/BusinessAttributeUtils.java @@ -0,0 +1,111 @@ +package com.linkedin.datahub.graphql.resolvers.mutate.util; + +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.schema.ArrayType; +import com.linkedin.schema.BooleanType; +import com.linkedin.schema.BytesType; +import com.linkedin.schema.DateType; +import com.linkedin.schema.EnumType; +import com.linkedin.schema.FixedType; +import com.linkedin.schema.MapType; +import com.linkedin.schema.NumberType; +import com.linkedin.schema.SchemaFieldDataType; +import com.linkedin.schema.StringType; +import com.linkedin.schema.TimeType; +import java.util.Objects; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class BusinessAttributeUtils { + private static final Integer DEFAULT_START = 0; + private static final Integer DEFAULT_COUNT = 1000; + private static final String NAME_INDEX_FIELD_NAME = "name"; + + private BusinessAttributeUtils() {} + + public static boolean hasNameConflict( + String name, QueryContext context, EntityClient entityClient) { + Filter filter = buildNameFilter(name); + try { + final SearchResult gmsResult = + entityClient.filter( + context.getOperationContext(), + Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME, + filter, + null, + DEFAULT_START, + DEFAULT_COUNT); + return gmsResult.getNumEntities() > 0; + } catch (RemoteInvocationException e) { + throw new RuntimeException("Failed to fetch Business Attributes", e); + } + } + + private static Filter buildNameFilter(String name) { + return new Filter() + .setOr( + new ConjunctiveCriterionArray( + new ConjunctiveCriterion().setAnd(buildNameCriterion(name)))); + } + + private static CriterionArray buildNameCriterion(@Nonnull final String name) { + return new CriterionArray( + new Criterion() + .setField(NAME_INDEX_FIELD_NAME) + .setValue(name) + .setCondition(Condition.EQUAL)); + } + + public static SchemaFieldDataType mapSchemaFieldDataType( + com.linkedin.datahub.graphql.generated.SchemaFieldDataType type) { + if (Objects.isNull(type)) { + return null; + } + SchemaFieldDataType schemaFieldDataType = new SchemaFieldDataType(); + switch (type) { + case BYTES: + schemaFieldDataType.setType(SchemaFieldDataType.Type.create(new BytesType())); + return schemaFieldDataType; + case FIXED: + schemaFieldDataType.setType(SchemaFieldDataType.Type.create(new FixedType())); + return schemaFieldDataType; + case ENUM: + schemaFieldDataType.setType(SchemaFieldDataType.Type.create(new EnumType())); + return schemaFieldDataType; + case MAP: + schemaFieldDataType.setType(SchemaFieldDataType.Type.create(new MapType())); + return schemaFieldDataType; + case TIME: + schemaFieldDataType.setType(SchemaFieldDataType.Type.create(new TimeType())); + return schemaFieldDataType; + case BOOLEAN: + schemaFieldDataType.setType(SchemaFieldDataType.Type.create(new BooleanType())); + return schemaFieldDataType; + case STRING: + schemaFieldDataType.setType(SchemaFieldDataType.Type.create(new StringType())); + return schemaFieldDataType; + case NUMBER: + schemaFieldDataType.setType(SchemaFieldDataType.Type.create(new NumberType())); + return schemaFieldDataType; + case DATE: + schemaFieldDataType.setType(SchemaFieldDataType.Type.create(new DateType())); + return schemaFieldDataType; + case ARRAY: + schemaFieldDataType.setType(SchemaFieldDataType.Type.create(new ArrayType())); + return schemaFieldDataType; + default: + return null; + } + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java index 09323fdfc83778..963d90c2e5692c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/LabelUtils.java @@ -5,6 +5,7 @@ import com.datahub.authorization.ConjunctivePrivilegeGroup; import com.datahub.authorization.DisjunctivePrivilegeGroup; import com.google.common.collect.ImmutableList; +import com.linkedin.businessattribute.BusinessAttributeInfo; import com.linkedin.common.GlobalTags; import com.linkedin.common.GlossaryTermAssociation; import com.linkedin.common.GlossaryTermAssociationArray; @@ -336,6 +337,10 @@ private static MetadataChangeProposal buildAddTagsProposal( throws URISyntaxException { if (resource.getSubResource() == null || resource.getSubResource().equals("")) { // Case 1: Adding tags to a top-level entity + Urn targetUrn = Urn.createFromString(resource.getResourceUrn()); + if (targetUrn.getEntityType().equals(Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME)) { + return buildAddTagsToBusinessAttributeProposal(tagUrns, resource, actor, entityService); + } return buildAddTagsToEntityProposal(tagUrns, resource, actor, entityService); } else { // Case 2: Adding tags to subresource (e.g. schema fields) @@ -348,6 +353,10 @@ private static MetadataChangeProposal buildRemoveTagsProposal( throws URISyntaxException { if (resource.getSubResource() == null || resource.getSubResource().equals("")) { // Case 1: Adding tags to a top-level entity + Urn targetUrn = Urn.createFromString(resource.getResourceUrn()); + if (targetUrn.getEntityType().equals(Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME)) { + return buildRemoveTagsToBusinessAttributeProposal(tagUrns, resource, actor, entityService); + } return buildRemoveTagsToEntityProposal(tagUrns, resource, actor, entityService); } else { // Case 2: Adding tags to subresource (e.g. schema fields) @@ -472,6 +481,10 @@ private static MetadataChangeProposal buildAddTermsProposal( throws URISyntaxException { if (resource.getSubResource() == null || resource.getSubResource().equals("")) { // Case 1: Adding terms to a top-level entity + Urn targetUrn = Urn.createFromString(resource.getResourceUrn()); + if (targetUrn.getEntityType().equals(Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME)) { + return buildAddTermsToBusinessAttributeProposal(termUrns, resource, actor, entityService); + } return buildAddTermsToEntityProposal(termUrns, resource, actor, entityService); } else { // Case 2: Adding terms to subresource (e.g. schema fields) @@ -484,6 +497,11 @@ private static MetadataChangeProposal buildRemoveTermsProposal( throws URISyntaxException { if (resource.getSubResource() == null || resource.getSubResource().equals("")) { // Case 1: Removing terms from a top-level entity + Urn targetUrn = Urn.createFromString(resource.getResourceUrn()); + if (targetUrn.getEntityType().equals(Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME)) { + return buildRemoveTermsToBusinessAttributeProposal( + termUrns, resource, actor, entityService); + } return buildRemoveTermsToEntityProposal(termUrns, resource, actor, entityService); } else { // Case 2: Removing terms from subresource (e.g. schema fields) @@ -615,4 +633,85 @@ private static GlossaryTermAssociationArray removeTermsIfExists( } return termAssociationArray; } + + private static MetadataChangeProposal buildAddTagsToBusinessAttributeProposal( + List tagUrns, ResourceRefInput resource, Urn actor, EntityService entityService) + throws URISyntaxException { + BusinessAttributeInfo businessAttributeInfo = + (BusinessAttributeInfo) + EntityUtils.getAspectFromEntity( + resource.getResourceUrn(), + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + entityService, + new GlobalTags()); + + if (!businessAttributeInfo.hasGlobalTags()) { + businessAttributeInfo.setGlobalTags(new GlobalTags()); + } + addTagsIfNotExists(businessAttributeInfo.getGlobalTags(), tagUrns); + return buildMetadataChangeProposalWithUrn( + UrnUtils.getUrn(resource.getResourceUrn()), + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + businessAttributeInfo); + } + + private static MetadataChangeProposal buildAddTermsToBusinessAttributeProposal( + List termUrns, ResourceRefInput resource, Urn actor, EntityService entityService) + throws URISyntaxException { + BusinessAttributeInfo businessAttributeInfo = + (BusinessAttributeInfo) + EntityUtils.getAspectFromEntity( + resource.getResourceUrn(), + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + entityService, + new GlossaryTerms()); + if (!businessAttributeInfo.hasGlossaryTerms()) { + businessAttributeInfo.setGlossaryTerms(new GlossaryTerms()); + } + businessAttributeInfo.getGlossaryTerms().setAuditStamp(EntityUtils.getAuditStamp(actor)); + addTermsIfNotExists(businessAttributeInfo.getGlossaryTerms(), termUrns); + return buildMetadataChangeProposalWithUrn( + UrnUtils.getUrn(resource.getResourceUrn()), + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + businessAttributeInfo); + } + + private static MetadataChangeProposal buildRemoveTagsToBusinessAttributeProposal( + List tagUrns, ResourceRefInput resource, Urn actor, EntityService entityService) { + BusinessAttributeInfo businessAttributeInfo = + (BusinessAttributeInfo) + EntityUtils.getAspectFromEntity( + resource.getResourceUrn(), + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + entityService, + new GlobalTags()); + + if (!businessAttributeInfo.hasGlobalTags()) { + businessAttributeInfo.setGlobalTags(new GlobalTags()); + } + removeTagsIfExists(businessAttributeInfo.getGlobalTags(), tagUrns); + return buildMetadataChangeProposalWithUrn( + UrnUtils.getUrn(resource.getResourceUrn()), + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + businessAttributeInfo); + } + + private static MetadataChangeProposal buildRemoveTermsToBusinessAttributeProposal( + List termUrns, ResourceRefInput resource, Urn actor, EntityService entityService) { + BusinessAttributeInfo businessAttributeInfo = + (BusinessAttributeInfo) + EntityUtils.getAspectFromEntity( + resource.getResourceUrn(), + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + entityService, + new GlossaryTerms()); + if (!businessAttributeInfo.hasGlossaryTerms()) { + businessAttributeInfo.setGlossaryTerms(new GlossaryTerms()); + } + removeTermsIfExists(businessAttributeInfo.getGlossaryTerms(), termUrns); + return buildMetadataChangeProposalWithUrn( + UrnUtils.getUrn(resource.getResourceUrn()), + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + businessAttributeInfo); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java index 3993e4a0db5999..10fb7cba56e3ff 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java @@ -71,7 +71,9 @@ private SearchUtils() {} EntityType.CONTAINER, EntityType.DOMAIN, EntityType.DATA_PRODUCT, - EntityType.NOTEBOOK); + EntityType.NOTEBOOK, + EntityType.BUSINESS_ATTRIBUTE, + EntityType.SCHEMA_FIELD); /** Entities that are part of autocomplete by default in Auto Complete Across Entities */ public static final List AUTO_COMPLETE_ENTITY_TYPES = @@ -91,7 +93,8 @@ private SearchUtils() {} EntityType.CORP_GROUP, EntityType.NOTEBOOK, EntityType.DATA_PRODUCT, - EntityType.DOMAIN); + EntityType.DOMAIN, + EntityType.BUSINESS_ATTRIBUTE); /** Entities that are part of browse by default */ public static final List BROWSE_ENTITY_TYPES = diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/businessattribute/BusinessAttributeType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/businessattribute/BusinessAttributeType.java new file mode 100644 index 00000000000000..5acfba5a1536ea --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/businessattribute/BusinessAttributeType.java @@ -0,0 +1,135 @@ +package com.linkedin.datahub.graphql.types.businessattribute; + +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME; +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_KEY_ASPECT_NAME; +import static com.linkedin.metadata.Constants.INSTITUTIONAL_MEMORY_ASPECT_NAME; +import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AutoCompleteResults; +import com.linkedin.datahub.graphql.generated.BusinessAttribute; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.SearchResults; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.types.SearchableEntityType; +import com.linkedin.datahub.graphql.types.businessattribute.mappers.BusinessAttributeMapper; +import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; +import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.query.AutoCompleteResult; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.SearchResult; +import graphql.execution.DataFetcherResult; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class BusinessAttributeType implements SearchableEntityType { + + public static final Set ASPECTS_TO_FETCH = + ImmutableSet.of( + BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + BUSINESS_ATTRIBUTE_KEY_ASPECT_NAME, + OWNERSHIP_ASPECT_NAME, + INSTITUTIONAL_MEMORY_ASPECT_NAME, + STATUS_ASPECT_NAME); + private static final Set FACET_FIELDS = ImmutableSet.of(""); + private final EntityClient _entityClient; + + @Override + public EntityType type() { + return EntityType.BUSINESS_ATTRIBUTE; + } + + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + + @Override + public Class objectClass() { + return BusinessAttribute.class; + } + + @Override + public List> batchLoad( + @Nonnull List urns, @Nonnull QueryContext context) throws Exception { + final List businessAttributeUrns = + urns.stream().map(UrnUtils::getUrn).collect(Collectors.toList()); + + try { + final Map businessAttributeMap = + _entityClient.batchGetV2( + BUSINESS_ATTRIBUTE_ENTITY_NAME, + new HashSet<>(businessAttributeUrns), + ASPECTS_TO_FETCH, + context.getAuthentication()); + + final List gmsResults = new ArrayList<>(); + for (Urn urn : businessAttributeUrns) { + gmsResults.add(businessAttributeMap.getOrDefault(urn, null)); + } + return gmsResults.stream() + .map( + gmsResult -> + gmsResult == null + ? null + : DataFetcherResult.newResult() + .data(BusinessAttributeMapper.map(context, gmsResult)) + .build()) + .collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Failed to batch load Business Attributes", e); + } + } + + @Override + public SearchResults search( + @Nonnull String query, + @Nullable List filters, + int start, + int count, + @Nonnull QueryContext context) + throws Exception { + final Map facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS); + final SearchResult searchResult = + _entityClient.search( + context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)), + "businessAttribute", + query, + facetFilters, + start, + count); + return UrnSearchResultsMapper.map(context, searchResult); + } + + @Override + public AutoCompleteResults autoComplete( + @Nonnull String query, + @Nullable String field, + @Nullable Filter filters, + int limit, + @Nonnull QueryContext context) + throws Exception { + final AutoCompleteResult result = + _entityClient.autoComplete( + context.getOperationContext(), "businessAttribute", query, filters, limit); + return AutoCompleteResultsMapper.map(context, result); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/businessattribute/mappers/BusinessAttributeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/businessattribute/mappers/BusinessAttributeMapper.java new file mode 100644 index 00000000000000..87230b24577163 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/businessattribute/mappers/BusinessAttributeMapper.java @@ -0,0 +1,134 @@ +package com.linkedin.datahub.graphql.types.businessattribute.mappers; + +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.INSTITUTIONAL_MEMORY_ASPECT_NAME; +import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME; + +import com.linkedin.businessattribute.BusinessAttributeInfo; +import com.linkedin.common.InstitutionalMemory; +import com.linkedin.common.Ownership; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.DataMap; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BusinessAttribute; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.SchemaFieldDataType; +import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; +import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; +import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; +import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class BusinessAttributeMapper implements ModelMapper { + + public static final BusinessAttributeMapper INSTANCE = new BusinessAttributeMapper(); + + public static BusinessAttribute map( + @Nullable final QueryContext context, @Nonnull final EntityResponse entityResponse) { + return INSTANCE.apply(context, entityResponse); + } + + @Override + public BusinessAttribute apply( + @Nullable final QueryContext context, @Nonnull final EntityResponse entityResponse) { + BusinessAttribute result = new BusinessAttribute(); + result.setUrn(entityResponse.getUrn().toString()); + result.setType(EntityType.BUSINESS_ATTRIBUTE); + + EnvelopedAspectMap aspectMap = entityResponse.getAspects(); + MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); + mappingHelper.mapToResult( + BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + ((businessAttribute, dataMap) -> + mapBusinessAttributeInfo( + context, businessAttribute, dataMap, entityResponse.getUrn()))); + mappingHelper.mapToResult( + OWNERSHIP_ASPECT_NAME, + (businessAttribute, dataMap) -> + businessAttribute.setOwnership( + OwnershipMapper.map(context, new Ownership(dataMap), entityResponse.getUrn()))); + mappingHelper.mapToResult( + INSTITUTIONAL_MEMORY_ASPECT_NAME, + (dataset, dataMap) -> + dataset.setInstitutionalMemory( + InstitutionalMemoryMapper.map( + context, new InstitutionalMemory(dataMap), entityResponse.getUrn()))); + return mappingHelper.getResult(); + } + + private void mapBusinessAttributeInfo( + final QueryContext context, + BusinessAttribute businessAttribute, + DataMap dataMap, + Urn entityUrn) { + BusinessAttributeInfo businessAttributeInfo = new BusinessAttributeInfo(dataMap); + com.linkedin.datahub.graphql.generated.BusinessAttributeInfo attributeInfo = + new com.linkedin.datahub.graphql.generated.BusinessAttributeInfo(); + if (businessAttributeInfo.hasFieldPath()) { + attributeInfo.setName(businessAttributeInfo.getFieldPath()); + } + if (businessAttributeInfo.hasDescription()) { + attributeInfo.setDescription(businessAttributeInfo.getDescription()); + } + if (businessAttributeInfo.hasCreated()) { + attributeInfo.setCreated(AuditStampMapper.map(context, businessAttributeInfo.getCreated())); + } + if (businessAttributeInfo.hasLastModified()) { + attributeInfo.setLastModified( + AuditStampMapper.map(context, businessAttributeInfo.getLastModified())); + } + if (businessAttributeInfo.hasGlobalTags()) { + attributeInfo.setTags( + GlobalTagsMapper.map(context, businessAttributeInfo.getGlobalTags(), entityUrn)); + } + if (businessAttributeInfo.hasGlossaryTerms()) { + attributeInfo.setGlossaryTerms( + GlossaryTermsMapper.map(context, businessAttributeInfo.getGlossaryTerms(), entityUrn)); + } + if (businessAttributeInfo.hasType()) { + attributeInfo.setType(mapSchemaFieldDataType(businessAttributeInfo.getType())); + } + if (businessAttributeInfo.hasCustomProperties()) { + attributeInfo.setCustomProperties( + CustomPropertiesMapper.map(businessAttributeInfo.getCustomProperties(), entityUrn)); + } + businessAttribute.setProperties(attributeInfo); + } + + private SchemaFieldDataType mapSchemaFieldDataType( + @Nonnull final com.linkedin.schema.SchemaFieldDataType dataTypeUnion) { + final com.linkedin.schema.SchemaFieldDataType.Type type = dataTypeUnion.getType(); + if (type.isBytesType()) { + return SchemaFieldDataType.BYTES; + } else if (type.isFixedType()) { + return SchemaFieldDataType.FIXED; + } else if (type.isBooleanType()) { + return SchemaFieldDataType.BOOLEAN; + } else if (type.isStringType()) { + return SchemaFieldDataType.STRING; + } else if (type.isNumberType()) { + return SchemaFieldDataType.NUMBER; + } else if (type.isDateType()) { + return SchemaFieldDataType.DATE; + } else if (type.isTimeType()) { + return SchemaFieldDataType.TIME; + } else if (type.isEnumType()) { + return SchemaFieldDataType.ENUM; + } else if (type.isArrayType()) { + return SchemaFieldDataType.ARRAY; + } else if (type.isMapType()) { + return SchemaFieldDataType.MAP; + } else { + throw new RuntimeException( + String.format( + "Unrecognized SchemaFieldDataType provided %s", type.memberType().toString())); + } + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/businessattribute/mappers/BusinessAttributesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/businessattribute/mappers/BusinessAttributesMapper.java new file mode 100644 index 00000000000000..104bc6ecd9222b --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/businessattribute/mappers/BusinessAttributesMapper.java @@ -0,0 +1,49 @@ +package com.linkedin.datahub.graphql.types.businessattribute.mappers; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.BusinessAttribute; +import com.linkedin.datahub.graphql.generated.BusinessAttributeAssociation; +import com.linkedin.datahub.graphql.generated.BusinessAttributes; +import com.linkedin.datahub.graphql.generated.EntityType; +import java.util.Objects; +import javax.annotation.Nonnull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BusinessAttributesMapper { + + private static final Logger _logger = + LoggerFactory.getLogger(BusinessAttributesMapper.class.getName()); + public static final BusinessAttributesMapper INSTANCE = new BusinessAttributesMapper(); + + public static BusinessAttributes map( + @Nonnull final com.linkedin.businessattribute.BusinessAttributes businessAttributes, + @Nonnull final Urn entityUrn) { + return INSTANCE.apply(businessAttributes, entityUrn); + } + + private BusinessAttributes apply( + @Nonnull com.linkedin.businessattribute.BusinessAttributes businessAttributes, + @Nonnull Urn entityUrn) { + final BusinessAttributes result = new BusinessAttributes(); + result.setBusinessAttribute( + mapBusinessAttributeAssociation(businessAttributes.getBusinessAttribute(), entityUrn)); + return result; + } + + private BusinessAttributeAssociation mapBusinessAttributeAssociation( + com.linkedin.businessattribute.BusinessAttributeAssociation businessAttributeAssociation, + Urn entityUrn) { + if (Objects.isNull(businessAttributeAssociation)) { + return null; + } + final BusinessAttributeAssociation businessAttributeAssociationResult = + new BusinessAttributeAssociation(); + final BusinessAttribute businessAttribute = new BusinessAttribute(); + businessAttribute.setUrn(businessAttributeAssociation.getBusinessAttributeUrn().toString()); + businessAttribute.setType(EntityType.BUSINESS_ATTRIBUTE); + businessAttributeAssociationResult.setBusinessAttribute(businessAttribute); + businessAttributeAssociationResult.setAssociatedUrn(entityUrn.toString()); + return businessAttributeAssociationResult; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java index 00f2a0df7512c3..1988cafc486c18 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java @@ -5,6 +5,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Assertion; +import com.linkedin.datahub.graphql.generated.BusinessAttribute; import com.linkedin.datahub.graphql.generated.Chart; import com.linkedin.datahub.graphql.generated.Container; import com.linkedin.datahub.graphql.generated.CorpGroup; @@ -219,6 +220,11 @@ public Entity apply(@Nullable QueryContext context, Urn input) { ((Restricted) partialEntity).setUrn(input.toString()); ((Restricted) partialEntity).setType(EntityType.RESTRICTED); } + if (input.getEntityType().equals(BUSINESS_ATTRIBUTE_ENTITY_NAME)) { + partialEntity = new BusinessAttribute(); + ((BusinessAttribute) partialEntity).setUrn(input.toString()); + ((BusinessAttribute) partialEntity).setType(EntityType.BUSINESS_ATTRIBUTE); + } return partialEntity; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java index e36d4e17f564da..48750082d3495c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java @@ -45,6 +45,7 @@ public class EntityTypeMapper { .put(EntityType.STRUCTURED_PROPERTY, Constants.STRUCTURED_PROPERTY_ENTITY_NAME) .put(EntityType.ASSERTION, Constants.ASSERTION_ENTITY_NAME) .put(EntityType.RESTRICTED, Constants.RESTRICTED_ENTITY_NAME) + .put(EntityType.BUSINESS_ATTRIBUTE, Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME) .build(); private static final Map ENTITY_NAME_TO_TYPE = diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java index 3d1833e9c944ad..85a6b9108cb54e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java @@ -1,11 +1,14 @@ package com.linkedin.datahub.graphql.types.schemafield; +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_ASPECT; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; +import com.linkedin.businessattribute.BusinessAttributes; import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.SchemaFieldEntity; +import com.linkedin.datahub.graphql.types.businessattribute.mappers.BusinessAttributesMapper; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; @@ -38,7 +41,11 @@ public SchemaFieldEntity apply( ((schemaField, dataMap) -> schemaField.setStructuredProperties( StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); - + mappingHelper.mapToResult( + BUSINESS_ATTRIBUTE_ASPECT, + (((schemaField, dataMap) -> + schemaField.setBusinessAttributes( + BusinessAttributesMapper.map(new BusinessAttributes(dataMap), entityUrn))))); return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldType.java index 6017f368eea240..bf9e044fdb1803 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldType.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.types.schemafield; +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_ASPECT; import static com.linkedin.metadata.Constants.SCHEMA_FIELD_ENTITY_NAME; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; @@ -31,7 +32,7 @@ public class SchemaFieldType implements com.linkedin.datahub.graphql.types.EntityType { public static final Set ASPECTS_TO_FETCH = - ImmutableSet.of(STRUCTURED_PROPERTIES_ASPECT_NAME); + ImmutableSet.of(STRUCTURED_PROPERTIES_ASPECT_NAME, BUSINESS_ATTRIBUTE_ASPECT); private final EntityClient _entityClient; private final FeatureFlags _featureFlags; diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index 39cff0f5114bfa..c8fb2dedd59284 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -130,6 +130,17 @@ type PlatformPrivileges { Whether the user can create and delete posts pinned to the home page. """ manageGlobalAnnouncements: Boolean! + + """ + Whether the user can create Business Attributes. + """ + createBusinessAttributes: Boolean! + + """ + Whether the user can manage Business Attributes. + """ + manageBusinessAttributes: Boolean! + } """ @@ -471,6 +482,11 @@ type FeatureFlagsConfig { If this is off, Domains appear "flat" again. """ nestedDomainsEnabled: Boolean! + + """ + Whether business attribute entity should be shown + """ + businessAttributeEntityEnabled: Boolean! } """ diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 4af8cc7f8c59ac..c4c82aa96a6000 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -236,6 +236,16 @@ type Query { Fetch a Data Platform Instance by primary key (urn) """ dataPlatformInstance(urn: String!): DataPlatformInstance + + """ + Fetch a Business Attribute by primary key (urn) + """ + businessAttribute(urn: String!): BusinessAttribute + + """ + Fetch all Business Attributes + """ + listBusinessAttributes(input: ListBusinessAttributesInput!): ListBusinessAttributesResult } """ @@ -889,6 +899,39 @@ type Mutation { is of type VERIFICATION. """ verifyForm(input: VerifyFormInput!): Boolean + + """ + Create Business Attribute Api + """ + createBusinessAttribute( + "Inputs required to create a new BusinessAttribute." + input: CreateBusinessAttributeInput!): BusinessAttribute + + """ + Delete a Business Attribute by urn. + """ + deleteBusinessAttribute( + "Urn of the business attribute to remove." + urn: String!): Boolean + + """ + Update Business Attribute + """ + updateBusinessAttribute( + "The urn identifier for the Business Attribute to update." + urn: String!, + "Inputs required to create a new Business Attribute." + input: UpdateBusinessAttributeInput!): BusinessAttribute + + """ + Add Business Attribute + """ + addBusinessAttribute(input: AddBusinessAttributeInput!): Boolean + + """ + Remove Business Attribute + """ + removeBusinessAttribute(input: AddBusinessAttributeInput!): Boolean } """ @@ -1139,6 +1182,11 @@ enum EntityType { Another entity type - refer to a provided entity type urn. """ OTHER + + """ + A Business Attribute + """ + BUSINESS_ATTRIBUTE } """ @@ -3147,6 +3195,11 @@ type SchemaFieldEntity implements Entity { Granular API for querying edges extending from this entity """ relationships(input: RelationshipsInput!): EntityRelationshipsResult + + """ + Business Attribute associated with the field + """ + businessAttributes: BusinessAttributes } """ @@ -3279,6 +3332,7 @@ type EditableSchemaFieldInfo { Glossary terms associated with the field """ glossaryTerms: GlossaryTerms + } """ @@ -12202,3 +12256,245 @@ type Restricted implements Entity & EntityWithRelationships { """ lineage(input: LineageInput!): EntityLineageResult } + + +""" +A Business Attribute, or a logical schema Field +""" +type BusinessAttribute implements Entity { + """ + The primary key of the Data Product + """ + urn: String! + + """ + A standard Entity Type + """ + type: EntityType! + + """ + Properties about a Business Attribute + """ + properties: BusinessAttributeInfo + + """ + Ownership metadata of the Business Attribute + """ + ownership: Ownership + + """ + References to internal resources related to Business Attribute + """ + institutionalMemory: InstitutionalMemory + + """ + Status of the Dataset + """ + status: Status + + """ + List of relationships between the source Entity and some destination entities with a given types + """ + relationships(input: RelationshipsInput!): EntityRelationshipsResult +} + +""" +Business Attribute type +""" + +type BusinessAttributeInfo { + + """ + name of the business attribute + """ + name: String! + + """ + description of business attribute + """ + description: String + + """ + Tags associated with the business attribute + """ + tags: GlobalTags + + """ + Glossary terms associated with the business attribute + """ + glossaryTerms: GlossaryTerms + + """ + Platform independent field type of the field + """ + type: SchemaFieldDataType + + """ + A list of platform specific metadata tuples + """ + customProperties: [CustomPropertiesEntry!] + + """ + An AuditStamp corresponding to the creation of this chart + """ + created: AuditStamp! + + """ + An AuditStamp corresponding to the modification of this chart + """ + lastModified: AuditStamp! + + """ + An optional AuditStamp corresponding to the deletion of this chart + """ + deleted: AuditStamp +} + +""" +Input required for creating a BusinessAttribute. +""" +input CreateBusinessAttributeInput { + """ + Optional! A custom id to use as the primary key identifier. If not provided, a random UUID will be generated as the id. + """ + id: String + + """ + name of the business attribute + """ + name: String! + + """ + description of business attribute + """ + description: String + + """ + Platform independent field type of the field + """ + type: SchemaFieldDataType + +} + +input BusinessAttributeInfoInput { + """ + name of the business attribute + """ + name: String! + + """ + description of business attribute + """ + description: String + + """ + Platform independent field type of the field + """ + type: SchemaFieldDataType +} + +""" +Input required to update Business Attribute +""" +input UpdateBusinessAttributeInput { + """ + name of the business attribute + """ + name: String + + """ + business attribute description + """ + description: String + + """ + type + """ + type: SchemaFieldDataType +} + +""" +Input required to attach Business Attribute +If businessAttributeUrn is null, then it will remove the business attribute from the resource +""" +input AddBusinessAttributeInput { + """ + The urn of the business attribute to add + """ + businessAttributeUrn: String! + + """ + resource urns to add the business attribute to + """ + resourceUrn: [ResourceRefInput!]! +} + +""" +Business attributes attached to the metadata +""" +type BusinessAttributes { + """ + Business Attribute attached to the Metadata Entity + """ + businessAttribute: BusinessAttributeAssociation +} + +""" +Input required to attach business attribute to an entity +""" +type BusinessAttributeAssociation { + """ + Business Attribute itself + """ + businessAttribute: BusinessAttribute! + + """ + Reference back to the associated urn for tracking purposes e.g. when sibling nodes are merged together + """ + associatedUrn: String! +} + +""" +Input provided when listing Business Attribute +""" +input ListBusinessAttributesInput { + """ + The starting offset of the result set returned + """ + start: Int + + """ + The maximum number of Business Attributes to be returned in the result set + """ + count: Int + + """ + Optional search query + """ + query: String +} + +""" +The result obtained when listing Business Attribute +""" +type ListBusinessAttributesResult { + """ + The starting offset of the result set returned + """ + start: Int! + + """ + The number of Business Attributes in the returned result set + """ + count: Int! + + """ + The total number of Business Attributes in the result set + """ + total: Int! + + """ + The Business Attributes + """ + businessAttributes: [BusinessAttribute!]! +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/AddBusinessAttributeResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/AddBusinessAttributeResolverTest.java new file mode 100644 index 00000000000000..f787879b7a0a60 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/AddBusinessAttributeResolverTest.java @@ -0,0 +1,130 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockEntityService; +import static org.mockito.ArgumentMatchers.eq; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + +import com.google.common.collect.ImmutableList; +import com.linkedin.businessattribute.BusinessAttributeAssociation; +import com.linkedin.businessattribute.BusinessAttributes; +import com.linkedin.common.urn.BusinessAttributeUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AddBusinessAttributeInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; +import graphql.schema.DataFetchingEnvironment; +import java.net.URISyntaxException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class AddBusinessAttributeResolverTest { + private static final String BUSINESS_ATTRIBUTE_URN = + "urn:li:businessAttribute:7d0c4283-de02-4043-aaf2-698b04274658"; + private static final String RESOURCE_URN = + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD),field_bar)"; + private EntityService mockService; + private QueryContext mockContext; + private DataFetchingEnvironment mockEnv; + + private void init() { + mockService = getMockEntityService(); + mockEnv = Mockito.mock(DataFetchingEnvironment.class); + } + + private void setupAllowContext() { + mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + } + + @Test + public void testSuccess() throws Exception { + init(); + setupAllowContext(); + + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(addBusinessAttributeInput()); + Mockito.when(mockService.exists(Urn.createFromString((BUSINESS_ATTRIBUTE_URN)), true)) + .thenReturn(true); + + Mockito.when( + mockService.getAspect( + Urn.createFromString(RESOURCE_URN), Constants.BUSINESS_ATTRIBUTE_ASPECT, 0)) + .thenReturn(new BusinessAttributes()); + + AddBusinessAttributeResolver addBusinessAttributeResolver = + new AddBusinessAttributeResolver(mockService); + addBusinessAttributeResolver.get(mockEnv).get(); + + Mockito.verify(mockService, Mockito.times(1)) + .ingestProposal(Mockito.any(AspectsBatchImpl.class), eq(false)); + } + + @Test + public void testBusinessAttributeAlreadyAdded() throws Exception { + init(); + setupAllowContext(); + + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(addBusinessAttributeInput()); + Mockito.when(mockService.exists(Urn.createFromString((BUSINESS_ATTRIBUTE_URN)), true)) + .thenReturn(true); + Mockito.when( + mockService.getAspect( + Urn.createFromString(RESOURCE_URN), Constants.BUSINESS_ATTRIBUTE_ASPECT, 0)) + .thenReturn(businessAttributes()); + + AddBusinessAttributeResolver addBusinessAttributeResolver = + new AddBusinessAttributeResolver(mockService); + addBusinessAttributeResolver.get(mockEnv).get(); + + Mockito.verify(mockService, Mockito.times(1)) + .ingestProposal(Mockito.any(AspectsBatchImpl.class), eq(false)); + } + + @Test + public void testBusinessAttributeNotExists() throws Exception { + init(); + setupAllowContext(); + + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(addBusinessAttributeInput()); + Mockito.when(mockService.exists(Urn.createFromString((BUSINESS_ATTRIBUTE_URN)), true)) + .thenReturn(false); + Mockito.when(mockService.exists(Urn.createFromString(RESOURCE_URN), true)).thenReturn(true); + + AddBusinessAttributeResolver addBusinessAttributeResolver = + new AddBusinessAttributeResolver(mockService); + RuntimeException exception = + expectThrows(RuntimeException.class, () -> addBusinessAttributeResolver.get(mockEnv).get()); + assertTrue( + exception + .getMessage() + .equals(String.format("This urn does not exist: %s", BUSINESS_ATTRIBUTE_URN))); + Mockito.verify(mockService, Mockito.times(0)) + .ingestProposal(Mockito.any(AspectsBatchImpl.class), eq(false)); + } + + public AddBusinessAttributeInput addBusinessAttributeInput() { + AddBusinessAttributeInput addBusinessAttributeInput = new AddBusinessAttributeInput(); + addBusinessAttributeInput.setBusinessAttributeUrn(BUSINESS_ATTRIBUTE_URN); + addBusinessAttributeInput.setResourceUrn(resourceRefInput()); + return addBusinessAttributeInput; + } + + private ImmutableList resourceRefInput() { + ResourceRefInput resourceRefInput = new ResourceRefInput(); + resourceRefInput.setResourceUrn(RESOURCE_URN); + return ImmutableList.of(resourceRefInput); + } + + private BusinessAttributes businessAttributes() throws URISyntaxException { + BusinessAttributes businessAttributes = new BusinessAttributes(); + BusinessAttributeAssociation businessAttributeAssociation = new BusinessAttributeAssociation(); + businessAttributeAssociation.setBusinessAttributeUrn( + BusinessAttributeUrn.createFromString(BUSINESS_ATTRIBUTE_URN)); + businessAttributes.setBusinessAttribute(businessAttributeAssociation); + return businessAttributes; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeProposalMatcher.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeProposalMatcher.java new file mode 100644 index 00000000000000..abed58aa883760 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeProposalMatcher.java @@ -0,0 +1,37 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import com.linkedin.businessattribute.BusinessAttributeInfo; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.GenericAspect; +import com.linkedin.mxe.MetadataChangeProposal; +import org.mockito.ArgumentMatcher; + +public class CreateBusinessAttributeProposalMatcher + implements ArgumentMatcher { + private MetadataChangeProposal left; + + public CreateBusinessAttributeProposalMatcher(MetadataChangeProposal left) { + this.left = left; + } + + @Override + public boolean matches(MetadataChangeProposal right) { + return left.getEntityType().equals(right.getEntityType()) + && left.getAspectName().equals(right.getAspectName()) + && left.getChangeType().equals(right.getChangeType()) + && businessAttributeInfoMatch(left.getAspect(), right.getAspect()); + } + + private boolean businessAttributeInfoMatch(GenericAspect left, GenericAspect right) { + BusinessAttributeInfo leftProps = + GenericRecordUtils.deserializeAspect( + left.getValue(), "application/json", BusinessAttributeInfo.class); + + BusinessAttributeInfo rightProps = + GenericRecordUtils.deserializeAspect( + right.getValue(), "application/json", BusinessAttributeInfo.class); + + return leftProps.getName().equals(rightProps.getName()) + && leftProps.getDescription().equals(rightProps.getDescription()); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeResolverTest.java new file mode 100644 index 00000000000000..e3dc2cb8a8f2f1 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/CreateBusinessAttributeResolverTest.java @@ -0,0 +1,250 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockEntityService; +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME; +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + +import com.datahub.authentication.Authentication; +import com.linkedin.businessattribute.BusinessAttributeInfo; +import com.linkedin.businessattribute.BusinessAttributeKey; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.CreateBusinessAttributeInput; +import com.linkedin.datahub.graphql.generated.SchemaFieldDataType; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.BusinessAttributeUtils; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.metadata.service.BusinessAttributeService; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.schema.BooleanType; +import graphql.schema.DataFetchingEnvironment; +import io.datahubproject.metadata.context.OperationContext; +import java.util.concurrent.ExecutionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class CreateBusinessAttributeResolverTest { + + private static final String BUSINESS_ATTRIBUTE_URN = + "urn:li:businessAttribute:business-attribute-1"; + private static final String TEST_BUSINESS_ATTRIBUTE_NAME = "test-business-attribute"; + private static final String TEST_BUSINESS_ATTRIBUTE_DESCRIPTION = "test-description"; + private static final CreateBusinessAttributeInput TEST_INPUT = + new CreateBusinessAttributeInput( + BUSINESS_ATTRIBUTE_URN, + TEST_BUSINESS_ATTRIBUTE_NAME, + TEST_BUSINESS_ATTRIBUTE_DESCRIPTION, + SchemaFieldDataType.BOOLEAN); + private static final CreateBusinessAttributeInput TEST_INPUT_NULL_NAME = + new CreateBusinessAttributeInput( + BUSINESS_ATTRIBUTE_URN, + null, + TEST_BUSINESS_ATTRIBUTE_DESCRIPTION, + SchemaFieldDataType.BOOLEAN); + private EntityClient mockClient; + private EntityService mockService; + private QueryContext mockContext; + private DataFetchingEnvironment mockEnv; + private BusinessAttributeService businessAttributeService; + private Authentication mockAuthentication; + private SearchResult searchResult; + + private void init() { + mockClient = Mockito.mock(EntityClient.class); + mockService = getMockEntityService(); + mockEnv = Mockito.mock(DataFetchingEnvironment.class); + businessAttributeService = Mockito.mock(BusinessAttributeService.class); + mockAuthentication = Mockito.mock(Authentication.class); + searchResult = Mockito.mock(SearchResult.class); + } + + @Test + public void testSuccess() throws Exception { + // Mock + init(); + setupAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockClient.exists(Mockito.any(Urn.class), Mockito.eq(mockAuthentication))) + .thenReturn(false); + Mockito.when( + mockClient.filter( + Mockito.any(OperationContext.class), + Mockito.any(String.class), + Mockito.any(Filter.class), + Mockito.isNull(), + Mockito.eq(0), + Mockito.eq(1000))) + .thenReturn(searchResult); + Mockito.when(searchResult.getNumEntities()).thenReturn(0); + Mockito.when( + mockClient.ingestProposal( + Mockito.any(MetadataChangeProposal.class), Mockito.eq(mockAuthentication))) + .thenReturn(BUSINESS_ATTRIBUTE_URN); + Mockito.when( + businessAttributeService.getBusinessAttributeEntityResponse( + Mockito.any(Urn.class), Mockito.eq(mockAuthentication))) + .thenReturn(getBusinessAttributeEntityResponse()); + + // Execute + CreateBusinessAttributeResolver resolver = + new CreateBusinessAttributeResolver(mockClient, mockService, businessAttributeService); + resolver.get(mockEnv).get(); + + // verify + Mockito.verify(mockClient, Mockito.times(1)) + .ingestProposal( + Mockito.argThat(new CreateBusinessAttributeProposalMatcher(metadataChangeProposal())), + Mockito.any(Authentication.class)); + } + + @Test + public void testNameIsNull() throws Exception { + // Mock + init(); + setupAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT_NULL_NAME); + Mockito.when(mockClient.exists(Mockito.any(Urn.class), Mockito.eq(mockAuthentication))) + .thenReturn(false); + + // Execute + CreateBusinessAttributeResolver resolver = + new CreateBusinessAttributeResolver(mockClient, mockService, businessAttributeService); + ExecutionException actualException = + expectThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + + // verify + assertTrue( + actualException + .getCause() + .getMessage() + .equals("Failed to create Business Attribute with name: null")); + + Mockito.verify(mockClient, Mockito.times(0)) + .ingestProposal( + Mockito.any(MetadataChangeProposal.class), Mockito.any(Authentication.class)); + } + + @Test + public void testNameAlreadyExists() throws Exception { + // Mock + init(); + setupAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockClient.exists(Mockito.any(Urn.class), Mockito.eq(mockAuthentication))) + .thenReturn(false); + Mockito.when( + mockClient.filter( + Mockito.any(OperationContext.class), + Mockito.any(String.class), + Mockito.any(Filter.class), + Mockito.isNull(), + Mockito.eq(0), + Mockito.eq(1000))) + .thenReturn(searchResult); + Mockito.when(searchResult.getNumEntities()).thenReturn(1); + + // Execute + CreateBusinessAttributeResolver resolver = + new CreateBusinessAttributeResolver(mockClient, mockService, businessAttributeService); + ExecutionException exception = + expectThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + + // Verify + assertTrue( + exception + .getCause() + .getMessage() + .equals( + "\"test-business-attribute\" already exists as Business Attribute. Please pick a unique name.")); + Mockito.verify(mockClient, Mockito.times(0)) + .ingestProposal( + Mockito.any(MetadataChangeProposal.class), Mockito.any(Authentication.class)); + } + + @Test + public void testUnauthorized() throws Exception { + init(); + setupDenyContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + + CreateBusinessAttributeResolver resolver = + new CreateBusinessAttributeResolver(mockClient, mockService, businessAttributeService); + AuthorizationException exception = + expectThrows(AuthorizationException.class, () -> resolver.get(mockEnv)); + + assertTrue( + exception + .getMessage() + .equals( + "Unauthorized to perform this action. Please contact your DataHub administrator.")); + Mockito.verify(mockClient, Mockito.times(0)) + .ingestProposal( + Mockito.any(MetadataChangeProposal.class), Mockito.any(Authentication.class)); + } + + private EntityResponse getBusinessAttributeEntityResponse() throws Exception { + EnvelopedAspectMap map = new EnvelopedAspectMap(); + BusinessAttributeInfo businessAttributeInfo = businessAttributeInfo(); + map.put( + BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(businessAttributeInfo.data()))); + EntityResponse entityResponse = new EntityResponse(); + entityResponse.setAspects(map); + entityResponse.setUrn(Urn.createFromString(BUSINESS_ATTRIBUTE_URN)); + return entityResponse; + } + + private MetadataChangeProposal metadataChangeProposal() { + BusinessAttributeKey businessAttributeKey = new BusinessAttributeKey(); + BusinessAttributeInfo info = new BusinessAttributeInfo(); + info.setFieldPath(TEST_BUSINESS_ATTRIBUTE_NAME); + info.setName(TEST_BUSINESS_ATTRIBUTE_NAME); + info.setDescription(TEST_BUSINESS_ATTRIBUTE_DESCRIPTION); + info.setType( + BusinessAttributeUtils.mapSchemaFieldDataType(SchemaFieldDataType.BOOLEAN), + SetMode.IGNORE_NULL); + return MutationUtils.buildMetadataChangeProposalWithKey( + businessAttributeKey, + BUSINESS_ATTRIBUTE_ENTITY_NAME, + BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + info); + } + + private void setupAllowContext() { + mockContext = getMockAllowContext(); + Mockito.when(mockContext.getAuthentication()).thenReturn(mockAuthentication); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + } + + private void setupDenyContext() { + mockContext = getMockDenyContext(); + Mockito.when(mockContext.getAuthentication()).thenReturn(mockAuthentication); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + } + + private BusinessAttributeInfo businessAttributeInfo() { + BusinessAttributeInfo businessAttributeInfo = new BusinessAttributeInfo(); + businessAttributeInfo.setName(TEST_BUSINESS_ATTRIBUTE_NAME); + businessAttributeInfo.setFieldPath(TEST_BUSINESS_ATTRIBUTE_NAME); + businessAttributeInfo.setDescription(TEST_BUSINESS_ATTRIBUTE_DESCRIPTION); + com.linkedin.schema.SchemaFieldDataType schemaFieldDataType = + new com.linkedin.schema.SchemaFieldDataType(); + schemaFieldDataType.setType( + com.linkedin.schema.SchemaFieldDataType.Type.create(new BooleanType())); + businessAttributeInfo.setType(schemaFieldDataType); + return businessAttributeInfo; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/DeleteBusinessAttributeResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/DeleteBusinessAttributeResolverTest.java new file mode 100644 index 00000000000000..114402a5b24dbf --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/DeleteBusinessAttributeResolverTest.java @@ -0,0 +1,106 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.entity.client.EntityClient; +import graphql.schema.DataFetchingEnvironment; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class DeleteBusinessAttributeResolverTest { + private static final String TEST_BUSINESS_ATTRIBUTE_URN = + "urn:li:businessAttribute:7d0c4283-de02-4043-aaf2-698b04274658"; + private EntityClient mockClient; + private QueryContext mockContext; + private DataFetchingEnvironment mockEnv; + private Authentication mockAuthentication; + + private void init() { + mockClient = Mockito.mock(EntityClient.class); + mockEnv = Mockito.mock(DataFetchingEnvironment.class); + mockAuthentication = Mockito.mock(Authentication.class); + } + + private void setupAllowContext() { + mockContext = getMockAllowContext(); + Mockito.when(mockContext.getAuthentication()).thenReturn(mockAuthentication); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + } + + private void setupDenyContext() { + mockContext = getMockDenyContext(); + Mockito.when(mockContext.getAuthentication()).thenReturn(mockAuthentication); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + } + + @Test + public void testSuccess() throws Exception { + init(); + setupAllowContext(); + Mockito.when(mockEnv.getArgument("urn")).thenReturn(TEST_BUSINESS_ATTRIBUTE_URN); + Mockito.when( + mockClient.exists( + Urn.createFromString(TEST_BUSINESS_ATTRIBUTE_URN), mockAuthentication)) + .thenReturn(true); + + DeleteBusinessAttributeResolver resolver = new DeleteBusinessAttributeResolver(mockClient); + resolver.get(mockEnv).get(); + + Mockito.verify(mockClient, Mockito.times(1)) + .deleteEntity( + Mockito.eq(Urn.createFromString(TEST_BUSINESS_ATTRIBUTE_URN)), + Mockito.any(Authentication.class)); + } + + @Test + public void testUnauthorized() throws Exception { + init(); + setupDenyContext(); + Mockito.when(mockEnv.getArgument("urn")).thenReturn(TEST_BUSINESS_ATTRIBUTE_URN); + + DeleteBusinessAttributeResolver resolver = new DeleteBusinessAttributeResolver(mockClient); + AuthorizationException actualException = + expectThrows(AuthorizationException.class, () -> resolver.get(mockEnv).get()); + assertTrue( + actualException + .getMessage() + .equals( + "Unauthorized to perform this action. Please contact your DataHub administrator.")); + + Mockito.verify(mockClient, Mockito.times(0)) + .deleteEntity( + Mockito.eq(Urn.createFromString(TEST_BUSINESS_ATTRIBUTE_URN)), + Mockito.any(Authentication.class)); + } + + @Test + public void testEntityNotExists() throws Exception { + init(); + setupAllowContext(); + Mockito.when(mockEnv.getArgument("urn")).thenReturn(TEST_BUSINESS_ATTRIBUTE_URN); + Mockito.when( + mockClient.exists( + Urn.createFromString(TEST_BUSINESS_ATTRIBUTE_URN), mockAuthentication)) + .thenReturn(false); + + DeleteBusinessAttributeResolver resolver = new DeleteBusinessAttributeResolver(mockClient); + RuntimeException actualException = + expectThrows(RuntimeException.class, () -> resolver.get(mockEnv).get()); + assertTrue( + actualException + .getMessage() + .equals(String.format("This urn does not exist: %s", TEST_BUSINESS_ATTRIBUTE_URN))); + + Mockito.verify(mockClient, Mockito.times(0)) + .deleteEntity( + Mockito.eq(Urn.createFromString(TEST_BUSINESS_ATTRIBUTE_URN)), + Mockito.any(Authentication.class)); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/RemoveBusinessAttributeResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/RemoveBusinessAttributeResolverTest.java new file mode 100644 index 00000000000000..78909a6910c13b --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/RemoveBusinessAttributeResolverTest.java @@ -0,0 +1,112 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockEntityService; +import static org.mockito.ArgumentMatchers.eq; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + +import com.google.common.collect.ImmutableList; +import com.linkedin.businessattribute.BusinessAttributeAssociation; +import com.linkedin.businessattribute.BusinessAttributes; +import com.linkedin.common.urn.BusinessAttributeUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AddBusinessAttributeInput; +import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; +import graphql.schema.DataFetchingEnvironment; +import java.net.URISyntaxException; +import java.util.concurrent.ExecutionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class RemoveBusinessAttributeResolverTest { + private static final String BUSINESS_ATTRIBUTE_URN = + "urn:li:businessAttribute:7d0c4283-de02-4043-aaf2-698b04274658"; + private static final String RESOURCE_URN = + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD),field_bar)"; + private EntityService mockService; + private QueryContext mockContext; + private DataFetchingEnvironment mockEnv; + + private void init() { + mockService = getMockEntityService(); + mockEnv = Mockito.mock(DataFetchingEnvironment.class); + } + + private void setupAllowContext() { + mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + } + + @Test + public void testSuccess() throws Exception { + init(); + setupAllowContext(); + + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(addBusinessAttributeInput()); + + Mockito.when( + mockService.getAspect( + Urn.createFromString(RESOURCE_URN), Constants.BUSINESS_ATTRIBUTE_ASPECT, 0)) + .thenReturn(businessAttributes()); + + RemoveBusinessAttributeResolver resolver = new RemoveBusinessAttributeResolver(mockService); + resolver.get(mockEnv).get(); + + Mockito.verify(mockService, Mockito.times(1)) + .ingestProposal(Mockito.any(AspectsBatchImpl.class), eq(false)); + } + + @Test + public void testBusinessAttributeNotAdded() throws Exception { + init(); + setupAllowContext(); + AddBusinessAttributeInput input = addBusinessAttributeInput(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); + Mockito.when( + mockService.getAspect( + Urn.createFromString(RESOURCE_URN), Constants.BUSINESS_ATTRIBUTE_ASPECT, 0)) + .thenReturn(new BusinessAttributes()); + + RemoveBusinessAttributeResolver resolver = new RemoveBusinessAttributeResolver(mockService); + ExecutionException actualException = + expectThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + assertTrue( + actualException + .getCause() + .getMessage() + .equals( + String.format( + "Failed to remove Business Attribute with urn %s from resources %s", + input.getBusinessAttributeUrn(), input.getResourceUrn()))); + + Mockito.verify(mockService, Mockito.times(0)) + .ingestProposal(Mockito.any(AspectsBatchImpl.class), eq(false)); + } + + public AddBusinessAttributeInput addBusinessAttributeInput() { + AddBusinessAttributeInput addBusinessAttributeInput = new AddBusinessAttributeInput(); + addBusinessAttributeInput.setBusinessAttributeUrn(BUSINESS_ATTRIBUTE_URN); + addBusinessAttributeInput.setResourceUrn(resourceRefInput()); + return addBusinessAttributeInput; + } + + private ImmutableList resourceRefInput() { + ResourceRefInput resourceRefInput = new ResourceRefInput(); + resourceRefInput.setResourceUrn(RESOURCE_URN); + return ImmutableList.of(resourceRefInput); + } + + private BusinessAttributes businessAttributes() throws URISyntaxException { + BusinessAttributes businessAttributes = new BusinessAttributes(); + BusinessAttributeAssociation businessAttributeAssociation = new BusinessAttributeAssociation(); + businessAttributeAssociation.setBusinessAttributeUrn( + BusinessAttributeUrn.createFromString(BUSINESS_ATTRIBUTE_URN)); + businessAttributes.setBusinessAttribute(businessAttributeAssociation); + return businessAttributes; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/UpdateBusinessAttributeResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/UpdateBusinessAttributeResolverTest.java new file mode 100644 index 00000000000000..44474956eec0b4 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/UpdateBusinessAttributeResolverTest.java @@ -0,0 +1,258 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + +import com.datahub.authentication.Authentication; +import com.linkedin.businessattribute.BusinessAttributeInfo; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.SchemaFieldDataType; +import com.linkedin.datahub.graphql.generated.UpdateBusinessAttributeInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.BusinessAttributeUtils; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.entity.AspectUtils; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.metadata.service.BusinessAttributeService; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.schema.BooleanType; +import graphql.schema.DataFetchingEnvironment; +import io.datahubproject.metadata.context.OperationContext; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class UpdateBusinessAttributeResolverTest { + private static final String TEST_BUSINESS_ATTRIBUTE_NAME = "test-business-attribute"; + private static final String TEST_BUSINESS_ATTRIBUTE_DESCRIPTION = "test-description"; + private static final String TEST_BUSINESS_ATTRIBUTE_NAME_UPDATED = + "test-business-attribute-updated"; + private static final String TEST_BUSINESS_ATTRIBUTE_DESCRIPTION_UPDATED = + "test-description-updated"; + private static final String TEST_BUSINESS_ATTRIBUTE_URN = + "urn:li:businessAttribute:7d0c4283-de02-4043-aaf2-698b04274658"; + private static final Urn TEST_BUSINESS_ATTRIBUTE_URN_OBJ = + UrnUtils.getUrn(TEST_BUSINESS_ATTRIBUTE_URN); + private EntityClient mockClient; + private QueryContext mockContext; + private DataFetchingEnvironment mockEnv; + private BusinessAttributeService businessAttributeService; + private Authentication mockAuthentication; + private SearchResult searchResult; + + private void init() { + mockClient = Mockito.mock(EntityClient.class); + mockEnv = Mockito.mock(DataFetchingEnvironment.class); + businessAttributeService = Mockito.mock(BusinessAttributeService.class); + mockAuthentication = Mockito.mock(Authentication.class); + searchResult = Mockito.mock(SearchResult.class); + } + + @Test + public void testSuccess() throws Exception { + init(); + setupAllowContext(); + final UpdateBusinessAttributeInput testInput = + new UpdateBusinessAttributeInput( + TEST_BUSINESS_ATTRIBUTE_NAME_UPDATED, + TEST_BUSINESS_ATTRIBUTE_DESCRIPTION_UPDATED, + SchemaFieldDataType.NUMBER); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getArgument("urn")).thenReturn(TEST_BUSINESS_ATTRIBUTE_URN); + Mockito.when(mockClient.exists(TEST_BUSINESS_ATTRIBUTE_URN_OBJ, mockAuthentication)) + .thenReturn(true); + Mockito.when( + businessAttributeService.getBusinessAttributeEntityResponse( + TEST_BUSINESS_ATTRIBUTE_URN_OBJ, mockAuthentication)) + .thenReturn(getBusinessAttributeEntityResponse()); + Mockito.when( + mockClient.filter( + Mockito.any(OperationContext.class), + Mockito.any(String.class), + Mockito.any(Filter.class), + Mockito.isNull(), + Mockito.eq(0), + Mockito.eq(1000))) + .thenReturn(searchResult); + Mockito.when(searchResult.getNumEntities()).thenReturn(0); + Mockito.when( + mockClient.ingestProposal( + Mockito.any(MetadataChangeProposal.class), Mockito.eq(mockAuthentication))) + .thenReturn(TEST_BUSINESS_ATTRIBUTE_URN); + + UpdateBusinessAttributeResolver resolver = + new UpdateBusinessAttributeResolver(mockClient, businessAttributeService); + resolver.get(mockEnv).get(); + + // verify + Mockito.verify(mockClient, Mockito.times(1)) + .ingestProposal( + Mockito.argThat( + new CreateBusinessAttributeProposalMatcher(updatedMetadataChangeProposal())), + Mockito.any(Authentication.class)); + } + + @Test + public void testNotExists() throws Exception { + init(); + setupAllowContext(); + final UpdateBusinessAttributeInput testInput = + new UpdateBusinessAttributeInput( + TEST_BUSINESS_ATTRIBUTE_NAME_UPDATED, + TEST_BUSINESS_ATTRIBUTE_DESCRIPTION_UPDATED, + SchemaFieldDataType.NUMBER); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getArgument("urn")).thenReturn(TEST_BUSINESS_ATTRIBUTE_URN); + Mockito.when(mockClient.exists(TEST_BUSINESS_ATTRIBUTE_URN_OBJ, mockAuthentication)) + .thenReturn(false); + + UpdateBusinessAttributeResolver resolver = + new UpdateBusinessAttributeResolver(mockClient, businessAttributeService); + RuntimeException expectedException = + expectThrows(RuntimeException.class, () -> resolver.get(mockEnv)); + assertTrue( + expectedException + .getMessage() + .equals(String.format("This urn does not exist: %s", TEST_BUSINESS_ATTRIBUTE_URN))); + + Mockito.verify(mockClient, Mockito.times(0)) + .ingestProposal( + Mockito.any(MetadataChangeProposal.class), Mockito.any(Authentication.class)); + } + + @Test + public void testNameConflict() throws Exception { + init(); + setupAllowContext(); + final UpdateBusinessAttributeInput testInput = + new UpdateBusinessAttributeInput( + TEST_BUSINESS_ATTRIBUTE_NAME, + TEST_BUSINESS_ATTRIBUTE_DESCRIPTION_UPDATED, + SchemaFieldDataType.NUMBER); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getArgument("urn")).thenReturn(TEST_BUSINESS_ATTRIBUTE_URN); + Mockito.when(mockClient.exists(TEST_BUSINESS_ATTRIBUTE_URN_OBJ, mockAuthentication)) + .thenReturn(true); + Mockito.when( + businessAttributeService.getBusinessAttributeEntityResponse( + TEST_BUSINESS_ATTRIBUTE_URN_OBJ, mockAuthentication)) + .thenReturn(getBusinessAttributeEntityResponse()); + Mockito.when( + mockClient.filter( + Mockito.any(OperationContext.class), + Mockito.any(String.class), + Mockito.any(Filter.class), + Mockito.isNull(), + Mockito.eq(0), + Mockito.eq(1000))) + .thenReturn(searchResult); + Mockito.when(searchResult.getNumEntities()).thenReturn(1); + + UpdateBusinessAttributeResolver resolver = + new UpdateBusinessAttributeResolver(mockClient, businessAttributeService); + + ExecutionException exception = + expectThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + + // Verify + assertTrue( + exception + .getCause() + .getMessage() + .equals( + "\"test-business-attribute\" already exists as Business Attribute. Please pick a unique name.")); + Mockito.verify(mockClient, Mockito.times(0)) + .ingestProposal( + Mockito.any(MetadataChangeProposal.class), Mockito.any(Authentication.class)); + } + + @Test + public void testNotAuthorized() throws Exception { + init(); + setupDenyContext(); + final UpdateBusinessAttributeInput testInput = + new UpdateBusinessAttributeInput( + TEST_BUSINESS_ATTRIBUTE_NAME, + TEST_BUSINESS_ATTRIBUTE_DESCRIPTION_UPDATED, + SchemaFieldDataType.NUMBER); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getArgument("urn")).thenReturn(TEST_BUSINESS_ATTRIBUTE_URN); + + UpdateBusinessAttributeResolver resolver = + new UpdateBusinessAttributeResolver(mockClient, businessAttributeService); + AuthorizationException exception = + expectThrows(AuthorizationException.class, () -> resolver.get(mockEnv)); + + assertTrue( + exception + .getMessage() + .equals( + "Unauthorized to perform this action. Please contact your DataHub administrator.")); + Mockito.verify(mockClient, Mockito.times(0)) + .ingestProposal( + Mockito.any(MetadataChangeProposal.class), Mockito.any(Authentication.class)); + } + + private EntityResponse getBusinessAttributeEntityResponse() throws Exception { + Map result = new HashMap<>(); + EnvelopedAspectMap map = new EnvelopedAspectMap(); + BusinessAttributeInfo businessAttributeInfo = businessAttributeInfo(); + map.put( + BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(businessAttributeInfo.data()))); + EntityResponse entityResponse = new EntityResponse(); + entityResponse.setAspects(map); + entityResponse.setUrn(Urn.createFromString(TEST_BUSINESS_ATTRIBUTE_URN)); + return entityResponse; + } + + private MetadataChangeProposal updatedMetadataChangeProposal() { + BusinessAttributeInfo info = new BusinessAttributeInfo(); + info.setFieldPath(TEST_BUSINESS_ATTRIBUTE_NAME_UPDATED); + info.setName(TEST_BUSINESS_ATTRIBUTE_NAME_UPDATED); + info.setDescription(TEST_BUSINESS_ATTRIBUTE_DESCRIPTION_UPDATED); + info.setType( + BusinessAttributeUtils.mapSchemaFieldDataType(SchemaFieldDataType.BOOLEAN), + SetMode.IGNORE_NULL); + return AspectUtils.buildMetadataChangeProposal( + TEST_BUSINESS_ATTRIBUTE_URN_OBJ, BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, info); + } + + private void setupAllowContext() { + mockContext = getMockAllowContext(); + Mockito.when(mockContext.getAuthentication()).thenReturn(mockAuthentication); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + } + + private void setupDenyContext() { + mockContext = getMockDenyContext(); + Mockito.when(mockContext.getAuthentication()).thenReturn(mockAuthentication); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + } + + private BusinessAttributeInfo businessAttributeInfo() { + BusinessAttributeInfo businessAttributeInfo = new BusinessAttributeInfo(); + businessAttributeInfo.setName(TEST_BUSINESS_ATTRIBUTE_NAME); + businessAttributeInfo.setFieldPath(TEST_BUSINESS_ATTRIBUTE_NAME); + businessAttributeInfo.setDescription(TEST_BUSINESS_ATTRIBUTE_DESCRIPTION); + com.linkedin.schema.SchemaFieldDataType schemaFieldDataType = + new com.linkedin.schema.SchemaFieldDataType(); + schemaFieldDataType.setType( + com.linkedin.schema.SchemaFieldDataType.Type.create(new BooleanType())); + businessAttributeInfo.setType(schemaFieldDataType); + return businessAttributeInfo; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/UpdateNameResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/UpdateNameResolverTest.java new file mode 100644 index 00000000000000..efc84c91409574 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/businessattribute/UpdateNameResolverTest.java @@ -0,0 +1,164 @@ +package com.linkedin.datahub.graphql.resolvers.businessattribute; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockEntityService; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + +import com.datahub.authentication.Authentication; +import com.linkedin.businessattribute.BusinessAttributeInfo; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.UpdateNameInput; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.UpdateNameResolver; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityUtils; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.schema.BooleanType; +import graphql.schema.DataFetchingEnvironment; +import io.datahubproject.metadata.context.OperationContext; +import java.util.concurrent.ExecutionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class UpdateNameResolverTest { + private static final String TEST_BUSINESS_ATTRIBUTE_NAME = "test-business-attribute"; + private static final String TEST_BUSINESS_ATTRIBUTE_NAME_UPDATED = + "test-business-attribute-updated"; + private static final String TEST_BUSINESS_ATTRIBUTE_DESCRIPTION = "test-description"; + private static final String TEST_BUSINESS_ATTRIBUTE_URN = + "urn:li:businessAttribute:7d0c4283-de02-4043-aaf2-698b04274658"; + private static final Urn TEST_BUSINESS_ATTRIBUTE_URN_OBJ = + UrnUtils.getUrn(TEST_BUSINESS_ATTRIBUTE_URN); + private EntityClient mockClient; + private EntityService mockService; + private QueryContext mockContext; + private DataFetchingEnvironment mockEnv; + private Authentication mockAuthentication; + private SearchResult searchResult; + + private void init() { + mockClient = Mockito.mock(EntityClient.class); + mockService = getMockEntityService(); + mockEnv = Mockito.mock(DataFetchingEnvironment.class); + mockAuthentication = Mockito.mock(Authentication.class); + searchResult = Mockito.mock(SearchResult.class); + } + + @Test + public void testSuccess() throws Exception { + init(); + setupAllowContext(); + UpdateNameInput testInput = + new UpdateNameInput(TEST_BUSINESS_ATTRIBUTE_NAME_UPDATED, TEST_BUSINESS_ATTRIBUTE_URN); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getArgument("urn")).thenReturn(TEST_BUSINESS_ATTRIBUTE_URN); + Mockito.when(mockService.exists(TEST_BUSINESS_ATTRIBUTE_URN_OBJ, true)).thenReturn(true); + Mockito.when( + EntityUtils.getAspectFromEntity( + TEST_BUSINESS_ATTRIBUTE_URN_OBJ.toString(), + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + mockService, + null)) + .thenReturn(businessAttributeInfo()); + + Mockito.when( + mockClient.filter( + Mockito.any(OperationContext.class), + Mockito.any(String.class), + Mockito.any(Filter.class), + Mockito.isNull(), + Mockito.eq(0), + Mockito.eq(1000))) + .thenReturn(searchResult); + Mockito.when(searchResult.getNumEntities()).thenReturn(0); + + BusinessAttributeInfo updatedBusinessAttributeInfo = businessAttributeInfo(); + updatedBusinessAttributeInfo.setName(TEST_BUSINESS_ATTRIBUTE_NAME_UPDATED); + updatedBusinessAttributeInfo.setFieldPath(TEST_BUSINESS_ATTRIBUTE_NAME_UPDATED); + MetadataChangeProposal proposal = + MutationUtils.buildMetadataChangeProposalWithUrn( + TEST_BUSINESS_ATTRIBUTE_URN_OBJ, + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + updatedBusinessAttributeInfo); + + UpdateNameResolver resolver = new UpdateNameResolver(mockService, mockClient); + resolver.get(mockEnv).get(); + + // verify + Mockito.verify(mockService, Mockito.times(1)) + .ingestProposal( + Mockito.argThat(new CreateBusinessAttributeProposalMatcher(proposal)), + Mockito.any(AuditStamp.class), + Mockito.eq(false)); + } + + @Test + public void testNameConflict() throws Exception { + init(); + setupAllowContext(); + UpdateNameInput testInput = + new UpdateNameInput(TEST_BUSINESS_ATTRIBUTE_NAME, TEST_BUSINESS_ATTRIBUTE_URN); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getArgument("urn")).thenReturn(TEST_BUSINESS_ATTRIBUTE_URN); + Mockito.when(mockService.exists(TEST_BUSINESS_ATTRIBUTE_URN_OBJ, true)).thenReturn(true); + Mockito.when( + EntityUtils.getAspectFromEntity( + TEST_BUSINESS_ATTRIBUTE_URN_OBJ.toString(), + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, + mockService, + null)) + .thenReturn(businessAttributeInfo()); + + Mockito.when( + mockClient.filter( + Mockito.any(OperationContext.class), + Mockito.any(String.class), + Mockito.any(Filter.class), + Mockito.isNull(), + Mockito.eq(0), + Mockito.eq(1000))) + .thenReturn(searchResult); + Mockito.when(searchResult.getNumEntities()).thenReturn(1); + + UpdateNameResolver resolver = new UpdateNameResolver(mockService, mockClient); + ExecutionException exception = + expectThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + + assertTrue( + exception + .getCause() + .getMessage() + .equals( + "\"test-business-attribute\" already exists as Business Attribute. Please pick a unique name.")); + Mockito.verify(mockClient, Mockito.times(0)) + .ingestProposal( + Mockito.any(MetadataChangeProposal.class), Mockito.any(Authentication.class)); + } + + private void setupAllowContext() { + mockContext = getMockAllowContext(); + Mockito.when(mockContext.getAuthentication()).thenReturn(mockAuthentication); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + } + + private BusinessAttributeInfo businessAttributeInfo() { + BusinessAttributeInfo businessAttributeInfo = new BusinessAttributeInfo(); + businessAttributeInfo.setName(TEST_BUSINESS_ATTRIBUTE_NAME); + businessAttributeInfo.setFieldPath(TEST_BUSINESS_ATTRIBUTE_NAME); + businessAttributeInfo.setDescription(TEST_BUSINESS_ATTRIBUTE_DESCRIPTION); + com.linkedin.schema.SchemaFieldDataType schemaFieldDataType = + new com.linkedin.schema.SchemaFieldDataType(); + schemaFieldDataType.setType( + com.linkedin.schema.SchemaFieldDataType.Type.create(new BooleanType())); + businessAttributeInfo.setType(schemaFieldDataType); + return businessAttributeInfo; + } +} diff --git a/datahub-graphql-core/src/test/resources/test-entity-registry.yaml b/datahub-graphql-core/src/test/resources/test-entity-registry.yaml index efd75a7fb07f51..4df822377ddf2b 100644 --- a/datahub-graphql-core/src/test/resources/test-entity-registry.yaml +++ b/datahub-graphql-core/src/test/resources/test-entity-registry.yaml @@ -293,6 +293,14 @@ entities: aspects: - ownershipTypeInfo - status +- name: businessAttribute + category: core + keyAspect: businessAttributeKey + aspects: + - businessAttributeInfo + - status + - ownership + - institutionalMemory - name: dataContract category: core keyAspect: dataContractKey @@ -300,4 +308,9 @@ entities: - dataContractProperties - dataContractStatus - status +- name: schemaField + category: core + keyAspect: schemaFieldKey + aspects: + - businessAttributes events: diff --git a/datahub-web-react/build.gradle b/datahub-web-react/build.gradle index 103792b20f761d..05af6871715ced 100644 --- a/datahub-web-react/build.gradle +++ b/datahub-web-react/build.gradle @@ -112,17 +112,13 @@ task yarnBuild(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) { outputs.dir('dist') } -task cleanGenerate { - delete fileTree(dir: 'src', include: '**/*.generated.ts') -} - task cleanExtraDirs { delete 'node_modules/.yarn-integrity' delete 'dist' delete 'tmp' delete 'just' + delete fileTree(dir: 'src', include: '*.generated.ts') } -cleanExtraDirs.finalizedBy(cleanGenerate) clean.finalizedBy(cleanExtraDirs) configurations { diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index 2d9d39c8ef46f0..c7e0a89ab38ea0 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -30,6 +30,7 @@ import { FilterOperator, AppConfig, EntityPrivileges, + BusinessAttribute, } from './types.generated'; import { GetTagDocument } from './graphql/tag.generated'; import { GetMlModelDocument } from './graphql/mlModel.generated'; @@ -1444,6 +1445,102 @@ export const dataJob1 = { health: [], } as DataJob; +export const businessAttribute = { + urn: 'urn:li:businessAttribute:ba1', + type: EntityType.BusinessAttribute, + __typename: 'BusinessAttribute', + properties: { + name: 'TestBusinessAtt-2', + description: 'lorem upsum updated 12', + created: { + time: 1705857132786 + }, + lastModified: { + time: 1705857132786 + }, + glossaryTerms: { + terms: [ + { + term: { + urn: 'urn:li:glossaryTerm:1', + type: EntityType.GlossaryTerm, + hierarchicalName: 'SampleHierarchicalName', + name: 'SampleName', + }, + associatedUrn: 'urn:li:businessAttribute:ba1' + } + ], + __typename: 'GlossaryTerms', + }, + tags: { + __typename: 'GlobalTags', + tags: [ + { + tag: { + urn: 'urn:li:tag:abc-sample-tag', + __typename: 'Tag', + type: EntityType.Tag, + name: 'abc-sample-tag', + }, + __typename: 'TagAssociation', + associatedUrn: 'urn:li:businessAttribute:ba1' + }, + { + tag: { + urn: 'urn:li:tag:TestTag', + __typename: 'Tag', + type: EntityType.Tag, + name: 'TestTag', + }, + __typename: 'TagAssociation', + associatedUrn: 'urn:li:businessAttribute:ba1' + } + ] + }, + customProperties: [ + { + key: 'prop2', + value: 'val2', + associatedUrn: 'urn:li:businessAttribute:ba1', + __typename: 'CustomPropertiesEntry' + }, + { + key: 'prop1', + value: 'val1', + associatedUrn: 'urn:li:businessAttribute:ba1', + __typename: 'CustomPropertiesEntry' + }, + { + key: 'prop3', + value: 'val3', + associatedUrn: 'urn:li:businessAttribute:ba1', + __typename: 'CustomPropertiesEntry' + } + ] + }, + ownership: { + owners: [ + { + owner: { + ...user1, + }, + associatedUrn: 'urn:li:businessAttribute:ba', + type: 'DATAOWNER', + }, + { + owner: { + ...user2, + }, + associatedUrn: 'urn:li:businessAttribute:ba', + type: 'DELEGATE', + }, + ], + lastModified: { + time: 0, + }, + }, +} as BusinessAttribute; + export const dataJob2 = { __typename: 'DataJob', urn: 'urn:li:dataJob:2', @@ -3528,6 +3625,8 @@ export const mocks = [ manageGlobalViews: true, manageOwnershipTypes: true, manageGlobalAnnouncements: true, + createBusinessAttributes: true, + manageBusinessAttributes: true, }, }, }, @@ -3802,4 +3901,6 @@ export const platformPrivileges: PlatformPrivileges = { manageGlobalViews: true, manageOwnershipTypes: true, manageGlobalAnnouncements: true, + createBusinessAttributes: true, + manageBusinessAttributes: true, }; diff --git a/datahub-web-react/src/app/SearchRoutes.tsx b/datahub-web-react/src/app/SearchRoutes.tsx index d2ad4ab6f4db19..4ebcc6f090a4bc 100644 --- a/datahub-web-react/src/app/SearchRoutes.tsx +++ b/datahub-web-react/src/app/SearchRoutes.tsx @@ -12,9 +12,9 @@ import { ManageIngestionPage } from './ingest/ManageIngestionPage'; import GlossaryRoutes from './glossary/GlossaryRoutes'; import { SettingsPage } from './settings/SettingsPage'; import DomainRoutes from './domain/DomainRoutes'; -import { useIsNestedDomainsEnabled } from './useAppConfig'; +import { useBusinessAttributesFlag, useIsAppConfigContextLoaded, useIsNestedDomainsEnabled } from './useAppConfig'; import { ManageDomainsPage } from './domain/ManageDomainsPage'; - +import { BusinessAttributes } from './businessAttribute/BusinessAttributes'; /** * Container for all searchable page routes */ @@ -25,6 +25,9 @@ export const SearchRoutes = (): JSX.Element => { ? entityRegistry.getEntitiesForSearchRoutes() : entityRegistry.getNonGlossaryEntities(); + const businessAttributesFlag = useBusinessAttributesFlag(); + const appConfigContextLoaded = useIsAppConfigContextLoaded(); + return ( @@ -50,6 +53,15 @@ export const SearchRoutes = (): JSX.Element => { } /> } /> } /> + { + if (!appConfigContextLoaded) { + return null; + } + if (businessAttributesFlag) { + return ; + } + return ; + }}/> diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts index f8e2534e44c310..d63b731c720426 100644 --- a/datahub-web-react/src/app/analytics/event.ts +++ b/datahub-web-react/src/app/analytics/event.ts @@ -81,6 +81,7 @@ export enum EventType { EmbedProfileViewEvent, EmbedProfileViewInDataHubEvent, EmbedLookupNotFoundEvent, + CreateBusinessAttributeEvent, } /** @@ -633,6 +634,11 @@ export interface EmbedLookupNotFoundEvent extends BaseEvent { reason: EmbedLookupNotFoundReason; } +export interface CreateBusinessAttributeEvent extends BaseEvent { + type: EventType.CreateBusinessAttributeEvent; + name: string; +} + /** * Event consisting of a union of specific event types. */ @@ -710,4 +716,5 @@ export type Event = | DeselectQuickFilterEvent | EmbedProfileViewEvent | EmbedProfileViewInDataHubEvent - | EmbedLookupNotFoundEvent; + | EmbedLookupNotFoundEvent + | CreateBusinessAttributeEvent; diff --git a/datahub-web-react/src/app/buildEntityRegistry.ts b/datahub-web-react/src/app/buildEntityRegistry.ts index 8e16bd52c62b17..ed207220830326 100644 --- a/datahub-web-react/src/app/buildEntityRegistry.ts +++ b/datahub-web-react/src/app/buildEntityRegistry.ts @@ -22,6 +22,8 @@ import { DataPlatformInstanceEntity } from './entity/dataPlatformInstance/DataPl import { ERModelRelationshipEntity } from './entity/ermodelrelationships/ERModelRelationshipEntity' import { RoleEntity } from './entity/Access/RoleEntity'; import { RestrictedEntity } from './entity/restricted/RestrictedEntity'; +import {BusinessAttributeEntity} from "./entity/businessAttribute/BusinessAttributeEntity"; +import { SchemaFieldPropertiesEntity } from './entity/schemaField/SchemaFieldPropertiesEntity'; export default function buildEntityRegistry() { const registry = new EntityRegistry(); @@ -48,5 +50,7 @@ export default function buildEntityRegistry() { registry.register(new DataPlatformInstanceEntity()); registry.register(new ERModelRelationshipEntity()) registry.register(new RestrictedEntity()); + registry.register(new BusinessAttributeEntity()); + registry.register(new SchemaFieldPropertiesEntity()); return registry; } \ No newline at end of file diff --git a/datahub-web-react/src/app/businessAttribute/AttributeBrowser.tsx b/datahub-web-react/src/app/businessAttribute/AttributeBrowser.tsx new file mode 100644 index 00000000000000..4d8f722aec9883 --- /dev/null +++ b/datahub-web-react/src/app/businessAttribute/AttributeBrowser.tsx @@ -0,0 +1,63 @@ +import React, { useEffect } from 'react'; +import styled from 'styled-components/macro'; +import { useEntityRegistry } from '../useEntityRegistry'; +import { ListBusinessAttributesQuery, useListBusinessAttributesQuery } from '../../graphql/businessAttribute.generated'; +import { sortBusinessAttributes } from './businessAttributeUtils'; +import AttributeItem from './AttributeItem'; + +const BrowserWrapper = styled.div` + color: #262626; + font-size: 12px; + max-height: calc(100% - 47px); + padding: 10px 20px 20px 20px; + overflow: auto; +`; + +interface Props { + isSelecting?: boolean; + hideTerms?: boolean; + refreshBrowser?: boolean; + selectAttribute?: (urn: string, displayName: string) => void; + attributeData?: ListBusinessAttributesQuery; +} + +function AttributeBrowser(props: Props) { + const { isSelecting, hideTerms, refreshBrowser, selectAttribute, attributeData } = props; + + const { refetch: refetchAttributes } = useListBusinessAttributesQuery({ + variables: { + start: 0, + count: 10, + query: '*', + }, + }); + + const displayedAttributes = attributeData?.listBusinessAttributes?.businessAttributes || []; + + const entityRegistry = useEntityRegistry(); + const sortedAttributes = displayedAttributes.sort((termA, termB) => + sortBusinessAttributes(entityRegistry, termA, termB), + ); + + useEffect(() => { + if (refreshBrowser) { + refetchAttributes(); + } + }, [refreshBrowser, refetchAttributes]); + + return ( + + {!hideTerms && + sortedAttributes.map((attribute) => ( + + ))} + + ); +} + +export default AttributeBrowser; diff --git a/datahub-web-react/src/app/businessAttribute/AttributeItem.tsx b/datahub-web-react/src/app/businessAttribute/AttributeItem.tsx new file mode 100644 index 00000000000000..051979d696f493 --- /dev/null +++ b/datahub-web-react/src/app/businessAttribute/AttributeItem.tsx @@ -0,0 +1,61 @@ +import React from 'react'; +import styled from 'styled-components/macro'; +import { ANTD_GRAY } from '../entity/shared/constants'; +import { useEntityRegistry } from '../useEntityRegistry'; + +const AttributeWrapper = styled.div` + font-weight: normal; + margin-bottom: 4px; +`; + +const nameStyles = ` + color: #262626; + display: inline-block; + height: 100%; + padding: 3px 4px; + width: 100%; +`; + +export const NameWrapper = styled.span<{ showSelectStyles?: boolean }>` + ${nameStyles} + + &:hover { + ${(props) => + props.showSelectStyles && + ` + background-color: ${ANTD_GRAY[3]}; + cursor: pointer; + `} + } +`; + +interface Props { + attribute: any; + isSelecting?: boolean; + selectAttribute?: (urn: string, displayName: string) => void; +} + +function AttributeItem(props: Props) { + const { attribute, isSelecting, selectAttribute } = props; + + const entityRegistry = useEntityRegistry(); + + function handleSelectAttribute() { + if (selectAttribute) { + const displayName = entityRegistry.getDisplayName(attribute.type, attribute); + selectAttribute(attribute.urn, displayName); + } + } + + return ( + + {isSelecting && ( + + {entityRegistry.getDisplayName(attribute.type, attribute)} + + )} + + ); +} + +export default AttributeItem; diff --git a/datahub-web-react/src/app/businessAttribute/BusinessAttributeItemMenu.tsx b/datahub-web-react/src/app/businessAttribute/BusinessAttributeItemMenu.tsx new file mode 100644 index 00000000000000..4e56d81203b6f5 --- /dev/null +++ b/datahub-web-react/src/app/businessAttribute/BusinessAttributeItemMenu.tsx @@ -0,0 +1,65 @@ +import React from 'react'; +import { DeleteOutlined } from '@ant-design/icons'; +import { Dropdown, Menu, message, Modal } from 'antd'; +import { MenuIcon } from '../entity/shared/EntityDropdown/EntityDropdown'; +import { useDeleteBusinessAttributeMutation } from '../../graphql/businessAttribute.generated'; + +type Props = { + urn: string; + title: string | undefined; + onDelete?: () => void; +}; + +export default function BusinessAttributeItemMenu({ title, urn, onDelete }: Props) { + const [deleteBusinessAttributeMutation] = useDeleteBusinessAttributeMutation(); + + const deletePost = () => { + deleteBusinessAttributeMutation({ + variables: { + urn, + }, + }) + .then(({ errors }) => { + if (!errors) { + message.success('Deleted Business Attribute!'); + onDelete?.(); + } + }) + .catch(() => { + message.destroy(); + message.error({ + content: `Failed to delete Business Attribute!: An unknown error occurred.`, + duration: 3, + }); + }); + }; + + const onConfirmDelete = () => { + Modal.confirm({ + title: `Delete Business Attribute '${title}'`, + content: `Are you sure you want to remove this Business Attribute?`, + onOk() { + deletePost(); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + }; + + return ( + + +  Delete + + + } + > + + + ); +} diff --git a/datahub-web-react/src/app/businessAttribute/BusinessAttributes.tsx b/datahub-web-react/src/app/businessAttribute/BusinessAttributes.tsx new file mode 100644 index 00000000000000..b16593f5497f6e --- /dev/null +++ b/datahub-web-react/src/app/businessAttribute/BusinessAttributes.tsx @@ -0,0 +1,257 @@ +import React, { useState, useMemo } from 'react'; +import styled from 'styled-components'; +import { Button, Empty, message, Pagination, Typography } from 'antd'; +import { PlusOutlined } from '@ant-design/icons'; +import { AlignType } from 'rc-table/lib/interface'; +import { Link } from 'react-router-dom'; +import { useListBusinessAttributesQuery } from '../../graphql/businessAttribute.generated'; +import { Message } from '../shared/Message'; +import TabToolbar from '../entity/shared/components/styled/TabToolbar'; +import { StyledTable } from '../entity/shared/components/styled/StyledTable'; +import CreateBusinessAttributeModal from './CreateBusinessAttributeModal'; +import { scrollToTop } from '../shared/searchUtils'; +import { useUserContext } from '../context/useUserContext'; +import { BusinessAttribute } from '../../types.generated'; +import { SearchBar } from '../search/SearchBar'; +import { useEntityRegistry } from '../useEntityRegistry'; +import useTagsAndTermsRenderer from './utils/useTagsAndTermsRenderer'; +import useDescriptionRenderer from './utils/useDescriptionRenderer'; +import BusinessAttributeItemMenu from './BusinessAttributeItemMenu'; + +function BusinessAttributeListMenuColumn(handleDelete: () => void) { + return (record: BusinessAttribute) => ( + handleDelete()} /> + ); +} + +const SourceContainer = styled.div` + width: 100%; + padding-top: 20px; + padding-right: 40px; + padding-left: 40px; + display: flex; + flex-direction: column; + overflow: auto; +`; + +const BusinessAttributesContainer = styled.div` + padding-top: 0px; +`; + +const BusinessAttributeHeaderContainer = styled.div` + && { + padding-left: 0px; + } +`; + +const BusinessAttributeTitle = styled(Typography.Title)` + && { + margin-bottom: 8px; + } +`; + +const PaginationContainer = styled.div` + display: flex; + justify-content: center; +`; + +const searchBarStyle = { + maxWidth: 220, + padding: 0, +}; + +const searchBarInputStyle = { + height: 32, + fontSize: 12, +}; + +const DEFAULT_PAGE_SIZE = 10; + +export const BusinessAttributes = () => { + const [isCreatingBusinessAttribute, setIsCreatingBusinessAttribute] = useState(false); + const entityRegistry = useEntityRegistry(); + + // Current User Urn + const authenticatedUser = useUserContext(); + + const canCreateBusinessAttributes = authenticatedUser?.platformPrivileges?.createBusinessAttributes; + const [page, setPage] = useState(1); + const pageSize = DEFAULT_PAGE_SIZE; + const start = (page - 1) * pageSize; + const [query, setQuery] = useState(undefined); + const [tagHoveredUrn, setTagHoveredUrn] = useState(undefined); + + const { + loading: businessAttributeLoading, + error: businessAttributeError, + data: businessAttributeData, + refetch: businessAttributeRefetch, + } = useListBusinessAttributesQuery({ + variables: { + start, + count: pageSize, + query, + }, + }); + const descriptionRender = useDescriptionRenderer(businessAttributeRefetch); + const tagRenderer = useTagsAndTermsRenderer( + tagHoveredUrn, + setTagHoveredUrn, + { + showTags: true, + showTerms: false, + }, + query || '', + businessAttributeRefetch, + ); + + const termRenderer = useTagsAndTermsRenderer( + tagHoveredUrn, + setTagHoveredUrn, + { + showTags: false, + showTerms: true, + }, + query || '', + businessAttributeRefetch, + ); + + const totalBusinessAttributes = businessAttributeData?.listBusinessAttributes?.total || 0; + const businessAttributes = useMemo( + () => (businessAttributeData?.listBusinessAttributes?.businessAttributes || []) as BusinessAttribute[], + [businessAttributeData], + ); + + const onTagTermCell = (record: BusinessAttribute) => ({ + onMouseEnter: () => { + setTagHoveredUrn(record.urn); + }, + onMouseLeave: () => { + setTagHoveredUrn(undefined); + }, + }); + + const handleDelete = () => { + setTimeout(() => { + businessAttributeRefetch?.(); + }, 2000); + }; + const tableData = businessAttributes || []; + const tableColumns = [ + { + width: '20%', + title: 'Name', + dataIndex: ['properties', 'name'], + key: 'name', + render: (name: string, record: any) => ( + {name} + ), + }, + { + title: 'Description', + dataIndex: ['properties', 'description'], + key: 'description', + width: '20%', + // render: (description: string) => description || '', + render: descriptionRender, + }, + { + width: '20%', + title: 'Tags', + dataIndex: ['properties', 'tags'], + key: 'tags', + render: tagRenderer, + onCell: onTagTermCell, + }, + { + width: '20%', + title: 'Glossary Terms', + dataIndex: ['properties', 'glossaryTags'], + key: 'glossaryTags', + render: termRenderer, + onCell: onTagTermCell, + }, + { + width: '13%', + title: 'Data Type', + dataIndex: ['properties', 'businessAttributeDataType'], + key: 'businessAttributeDataType', + render: (dataType: string) => dataType || '', + }, + { + title: '', + dataIndex: '', + width: '5%', + align: 'right' as AlignType, + key: 'menu', + render: BusinessAttributeListMenuColumn(handleDelete), + }, + ]; + + const onChangePage = (newPage: number) => { + scrollToTop(); + setPage(newPage); + }; + + return ( + + {businessAttributeLoading && !businessAttributeData && ( + + )} + {businessAttributeError && message.error('Failed to load businessAttributes :(')} + + + Business Attribute + View your Business Attributes + + + + + null} + onQueryChange={(q) => setQuery(q.length > 0 ? q : undefined)} + entityRegistry={entityRegistry} + /> + + , + }} + pagination={false} + /> + + + + setIsCreatingBusinessAttribute(false)} + onCreateBusinessAttribute={() => { + businessAttributeRefetch?.(); + }} + /> + + ); +}; diff --git a/datahub-web-react/src/app/businessAttribute/CreateBusinessAttributeModal.tsx b/datahub-web-react/src/app/businessAttribute/CreateBusinessAttributeModal.tsx new file mode 100644 index 00000000000000..61595045646c4b --- /dev/null +++ b/datahub-web-react/src/app/businessAttribute/CreateBusinessAttributeModal.tsx @@ -0,0 +1,255 @@ +import React, { useState } from 'react'; +import { message, Button, Input, Modal, Typography, Form, Select, Collapse } from 'antd'; +import styled from 'styled-components'; +import { EditOutlined } from '@ant-design/icons'; +import DOMPurify from 'dompurify'; +import { useEnterKeyListener } from '../shared/useEnterKeyListener'; +import { useCreateBusinessAttributeMutation } from '../../graphql/businessAttribute.generated'; +import { CreateBusinessAttributeInput, EntityType } from '../../types.generated'; +import analytics, { EventType } from '../analytics'; +import { useEntityRegistry } from '../useEntityRegistry'; +import DescriptionModal from '../entity/shared/components/legacy/DescriptionModal'; +import { SchemaFieldDataType } from './businessAttributeUtils'; +import { validateCustomUrnId } from '../shared/textUtil'; + +type Props = { + visible: boolean; + onClose: () => void; + onCreateBusinessAttribute: () => void; +}; + +type FormProps = { + name: string; + description?: string; + dataType?: SchemaFieldDataType; +}; + +const DataTypeSelectContainer = styled.div` + padding: 1px; +`; + +const DataTypeSelect = styled(Select)` + && { + width: 100%; + margin-top: 1em; + margin-bottom: 1em; + } +`; + +const StyledItem = styled(Form.Item)` + margin-bottom: 0; +`; + +const OptionalWrapper = styled.span` + font-weight: normal; +`; + +const StyledButton = styled(Button)` + padding: 0; +`; + +// Ensures that any newly added datatype is automatically included in the user dropdown. +const DATA_TYPES = Object.values(SchemaFieldDataType); + +export default function CreateBusinessAttributeModal({ visible, onClose, onCreateBusinessAttribute }: Props) { + const [createButtonEnabled, setCreateButtonEnabled] = useState(true); + + const [createBusinessAttribute] = useCreateBusinessAttributeMutation(); + + const [isDocumentationModalVisible, setIsDocumentationModalVisible] = useState(false); + + const [documentation, setDocumentation] = useState(''); + + const [form] = Form.useForm(); + + const entityRegistry = useEntityRegistry(); + + const [stagedId, setStagedId] = useState(undefined); + + // Function to handle the close or cross button of Create Business Attribute Modal + const onModalClose = () => { + form.resetFields(); + onClose(); + }; + + const onCreateNewBusinessAttribute = () => { + const { name, dataType } = form.getFieldsValue(); + const sanitizedDescription = DOMPurify.sanitize(documentation); + const input: CreateBusinessAttributeInput = { + id: stagedId?.length ? stagedId : undefined, + name, + description: sanitizedDescription, + type: dataType, + }; + createBusinessAttribute({ variables: { input } }) + .then(() => { + message.loading({ content: 'Updating...', duration: 2 }); + setTimeout(() => { + analytics.event({ + type: EventType.CreateBusinessAttributeEvent, + name, + }); + message.success({ + content: `Created ${entityRegistry.getEntityName(EntityType.BusinessAttribute)}!`, + duration: 2, + }); + if (onCreateBusinessAttribute) { + onCreateBusinessAttribute(); + } + }, 2000); + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to create: \n ${e.message || ''}`, duration: 3 }); + }); + onModalClose(); + setDocumentation(''); + }; + + // Handle the Enter press + useEnterKeyListener({ + querySelectorToExecuteClick: '#createBusinessAttributeButton', + }); + + function addDocumentation(description: string) { + setDocumentation(description); + setIsDocumentationModalVisible(false); + } + + return ( + <> + + + + + } + > +
+ setCreateButtonEnabled(form.getFieldsError().some((field) => field.errors.length > 0)) + } + > + Name}> + + + + + + Data Type}> + + + {DATA_TYPES.map((dataType: SchemaFieldDataType) => ( + + {dataType} + + ))} + + + + + + Documentation (optional) + + } + > + setIsDocumentationModalVisible(true)}> + + {documentation ? 'Edit' : 'Add'} Documentation + + {isDocumentationModalVisible && ( + setIsDocumentationModalVisible(false)} + onSubmit={addDocumentation} + description={documentation} + /> + )} + + + Advanced} key="1"> + + {entityRegistry.getEntityName(EntityType.BusinessAttribute)} Id + + } + > + + By default, a random UUID will be generated to uniquely identify this entity. If + you'd like to provide a custom id, you may provide it here. Note that it should be + unique across the entire Business Attributes. Be careful, you cannot easily change the id after + creation. + + ({ + validator(_, value) { + if (value && validateCustomUrnId(value)) { + return Promise.resolve(); + } + return Promise.reject(new Error('Please enter a valid entity id')); + }, + }), + ]} + > + setStagedId(event.target.value)} + /> + + + + +
+ + + ); +} diff --git a/datahub-web-react/src/app/businessAttribute/businessAttributeUtils.ts b/datahub-web-react/src/app/businessAttribute/businessAttributeUtils.ts new file mode 100644 index 00000000000000..ec8c44d79901c3 --- /dev/null +++ b/datahub-web-react/src/app/businessAttribute/businessAttributeUtils.ts @@ -0,0 +1,37 @@ +import EntityRegistry from '../entity/EntityRegistry'; +import { Entity, EntityType } from '../../types.generated'; + +export function sortBusinessAttributes(entityRegistry: EntityRegistry, nodeA?: Entity | null, nodeB?: Entity | null) { + const nodeAName = entityRegistry.getDisplayName(EntityType.BusinessAttribute, nodeA) || ''; + const nodeBName = entityRegistry.getDisplayName(EntityType.BusinessAttribute, nodeB) || ''; + return nodeAName.localeCompare(nodeBName); +} + +export function getRelatedEntitiesUrl(entityRegistry: EntityRegistry, urn: string) { + return `${entityRegistry.getEntityUrl(EntityType.BusinessAttribute, urn)}/${encodeURIComponent( + 'Related Entities', + )}`; +} + +export enum SchemaFieldDataType { + /** A boolean type */ + Boolean = 'BOOLEAN', + /** A fixed bytestring type */ + Fixed = 'FIXED', + /** A string type */ + String = 'STRING', + /** A string of bytes */ + Bytes = 'BYTES', + /** A number, including integers, floats, and doubles */ + Number = 'NUMBER', + /** A datestrings type */ + Date = 'DATE', + /** A timestamp type */ + Time = 'TIME', + /** An enum type */ + Enum = 'ENUM', + /** A map collection type */ + Map = 'MAP', + /** An array collection type */ + Array = 'ARRAY', +} diff --git a/datahub-web-react/src/app/businessAttribute/utils/useDescriptionRenderer.tsx b/datahub-web-react/src/app/businessAttribute/utils/useDescriptionRenderer.tsx new file mode 100644 index 00000000000000..ef665e45aeefdd --- /dev/null +++ b/datahub-web-react/src/app/businessAttribute/utils/useDescriptionRenderer.tsx @@ -0,0 +1,41 @@ +import React, { useState } from 'react'; +import DOMPurify from 'dompurify'; +import { BusinessAttribute } from '../../../types.generated'; +import DescriptionField from '../../entity/dataset/profile/schema/components/SchemaDescriptionField'; +import { useUpdateDescriptionMutation } from '../../../graphql/mutations.generated'; + +export default function useDescriptionRenderer(businessAttributeRefetch: () => Promise) { + const [updateDescription] = useUpdateDescriptionMutation(); + const [expandedRows, setExpandedRows] = useState({}); + + const refresh: any = () => { + businessAttributeRefetch?.(); + }; + + return (description: string, record: BusinessAttribute, index: number): JSX.Element => { + const relevantEditableFieldInfo = record?.properties; + const displayedDescription = relevantEditableFieldInfo?.description || description; + const sanitizedDescription = DOMPurify.sanitize(displayedDescription); + + const handleExpandedRows = (expanded) => setExpandedRows((prev) => ({ ...prev, [index]: expanded })); + + return ( + + updateDescription({ + variables: { + input: { + description: DOMPurify.sanitize(updatedDescription), + resourceUrn: record.urn, + }, + }, + }).then(refresh) + } + /> + ); + }; +} +// diff --git a/datahub-web-react/src/app/businessAttribute/utils/useTagsAndTermsRenderer.tsx b/datahub-web-react/src/app/businessAttribute/utils/useTagsAndTermsRenderer.tsx new file mode 100644 index 00000000000000..7c138c99dbd1a8 --- /dev/null +++ b/datahub-web-react/src/app/businessAttribute/utils/useTagsAndTermsRenderer.tsx @@ -0,0 +1,38 @@ +import React from 'react'; +import { EntityType, GlobalTags, BusinessAttribute } from '../../../types.generated'; +import TagTermGroup from '../../shared/tags/TagTermGroup'; + +export default function useTagsAndTermsRenderer( + tagHoveredUrn: string | undefined, + setTagHoveredUrn: (index: string | undefined) => void, + options: { showTags: boolean; showTerms: boolean }, + filterText: string, + businessAttributeRefetch: () => Promise, +) { + const urn = tagHoveredUrn; + + const refresh: any = () => { + businessAttributeRefetch?.(); + }; + + const tagAndTermRender = (tags: GlobalTags, record: BusinessAttribute) => { + return ( +
+ setTagHoveredUrn(undefined)} + entityUrn={urn} + entityType={EntityType.BusinessAttribute} + highlightText={filterText} + refetch={refresh} + /> +
+ ); + }; + return tagAndTermRender; +} diff --git a/datahub-web-react/src/app/entity/Entity.tsx b/datahub-web-react/src/app/entity/Entity.tsx index 613eba3d648c29..490f23330c5945 100644 --- a/datahub-web-react/src/app/entity/Entity.tsx +++ b/datahub-web-react/src/app/entity/Entity.tsx @@ -81,6 +81,10 @@ export enum EntityCapabilityType { * Assigning the entity to a data product */ DATA_PRODUCTS, + /** + * Assigning Business Attribute to a entity + */ + BUSINESS_ATTRIBUTES, } /** @@ -199,4 +203,9 @@ export interface Entity { urn: string; }> >; + + /** + * Returns the url to be navigated to when clicked on Cards + */ + getCustomCardUrlPath?: () => string | undefined; } diff --git a/datahub-web-react/src/app/entity/EntityRegistry.tsx b/datahub-web-react/src/app/entity/EntityRegistry.tsx index b2cb63c6e0a71b..00e7385ff5784b 100644 --- a/datahub-web-react/src/app/entity/EntityRegistry.tsx +++ b/datahub-web-react/src/app/entity/EntityRegistry.tsx @@ -236,4 +236,9 @@ export default class EntityRegistry { .map((entity) => entity.type), ); } + + getCustomCardUrlPath(type: EntityType): string | undefined { + const entity = validatedGet(type, this.entityTypeToEntity); + return entity.getCustomCardUrlPath?.(); + } } diff --git a/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx b/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx new file mode 100644 index 00000000000000..b827a3c37d6a5c --- /dev/null +++ b/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx @@ -0,0 +1,156 @@ +import * as React from 'react'; +import { GlobalOutlined } from '@ant-design/icons'; +import { BusinessAttribute, EntityType, SearchResult } from '../../../types.generated'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; +import { getDataForEntityType } from '../shared/containers/profile/utils'; +import { EntityProfile } from '../shared/containers/profile/EntityProfile'; +import { useGetBusinessAttributeQuery } from '../../../graphql/businessAttribute.generated'; +import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; +import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'; +import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab'; +import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutSection/SidebarAboutSection'; +import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection'; +import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection'; +import { Preview } from './preview/Preview'; +import { PageRoutes } from '../../../conf/Global'; +import BusinessAttributeRelatedEntity from './profile/BusinessAttributeRelatedEntity'; +import { BusinessAttributeDataTypeSection } from './profile/BusinessAttributeDataTypeSection'; + +/** + * Definition of datahub Business Attribute Entity + */ +/* eslint-disable @typescript-eslint/no-unused-vars */ +export class BusinessAttributeEntity implements Entity { + type: EntityType = EntityType.BusinessAttribute; + + icon = (fontSize: number, styleType: IconStyleType, color?: string) => { + if (styleType === IconStyleType.TAB_VIEW) { + return ; + } + + if (styleType === IconStyleType.HIGHLIGHT) { + return ; + } + + if (styleType === IconStyleType.SVG) { + // TODO: Update the returned path value to the correct svg icon path + return ( + + ); + } + + return ( + + ); + }; + + displayName = (data: BusinessAttribute) => { + return data?.properties?.name || data?.urn; + }; + + getPathName = () => 'business-attribute'; + + getEntityName = () => 'Business Attribute'; + + getCollectionName = () => 'Business Attributes'; + + getCustomCardUrlPath = () => PageRoutes.BUSINESS_ATTRIBUTE; + + isBrowseEnabled = () => false; + + isLineageEnabled = () => false; + + isSearchEnabled = () => true; + + getOverridePropertiesFromEntity = (data: BusinessAttribute) => { + return { + name: data.properties?.name, + }; + }; + + getGenericEntityProperties = (data: BusinessAttribute) => { + return getDataForEntityType({ + data, + entityType: this.type, + getOverrideProperties: this.getOverridePropertiesFromEntity, + }); + }; + + renderPreview = (previewType: PreviewType, data: BusinessAttribute) => { + return ( + + ); + }; + + renderProfile = (urn: string) => { + return ( + + ); + }; + + renderSearch = (result: SearchResult) => { + return this.renderPreview(PreviewType.SEARCH, result.entity as BusinessAttribute); + }; + + supportedCapabilities = () => { + return new Set([ + EntityCapabilityType.OWNERS, + EntityCapabilityType.TAGS, + EntityCapabilityType.GLOSSARY_TERMS, + EntityCapabilityType.BUSINESS_ATTRIBUTES, + ]); + }; +} diff --git a/datahub-web-react/src/app/entity/businessAttribute/preview/Preview.tsx b/datahub-web-react/src/app/entity/businessAttribute/preview/Preview.tsx new file mode 100644 index 00000000000000..323c287a0acd78 --- /dev/null +++ b/datahub-web-react/src/app/entity/businessAttribute/preview/Preview.tsx @@ -0,0 +1,40 @@ +import React from 'react'; +import { GlobalOutlined } from '@ant-design/icons'; +import { EntityType, Owner } from '../../../../types.generated'; +import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import { IconStyleType, PreviewType } from '../../Entity'; +import UrlButton from '../../shared/UrlButton'; +import { getRelatedEntitiesUrl } from '../../../businessAttribute/businessAttributeUtils'; + +export const Preview = ({ + urn, + name, + description, + owners, + previewType, +}: { + urn: string; + name: string; + description?: string | null; + owners?: Array | null; + previewType: PreviewType; +}): JSX.Element => { + const entityRegistry = useEntityRegistry(); + return ( + } + type="Business Attribute" + typeIcon={entityRegistry.getIcon(EntityType.BusinessAttribute, 14, IconStyleType.ACCENT)} + entityTitleSuffix={ + View Related Entities + } + /> + ); +}; diff --git a/datahub-web-react/src/app/entity/businessAttribute/preview/_tests_/Preview.test.tsx b/datahub-web-react/src/app/entity/businessAttribute/preview/_tests_/Preview.test.tsx new file mode 100644 index 00000000000000..51a6db654129f9 --- /dev/null +++ b/datahub-web-react/src/app/entity/businessAttribute/preview/_tests_/Preview.test.tsx @@ -0,0 +1,26 @@ +import {MockedProvider} from '@apollo/client/testing'; +import {render} from '@testing-library/react'; +import React from 'react'; +import {mocks} from '../../../../../Mocks'; +import TestPageContainer from '../../../../../utils/test-utils/TestPageContainer'; +import {Preview} from '../Preview'; +import {PreviewType} from "../../../Entity"; + +describe('Preview', () => { + it('renders', () => { + const { getByText } = render( + + + + + , + ); + expect(getByText('definition')).toBeInTheDocument(); + }); +}); diff --git a/datahub-web-react/src/app/entity/businessAttribute/profile/BusinessAttributeDataTypeSection.tsx b/datahub-web-react/src/app/entity/businessAttribute/profile/BusinessAttributeDataTypeSection.tsx new file mode 100644 index 00000000000000..da2b108c2d8d04 --- /dev/null +++ b/datahub-web-react/src/app/entity/businessAttribute/profile/BusinessAttributeDataTypeSection.tsx @@ -0,0 +1,98 @@ +import { Button, message, Select } from 'antd'; +import { EditOutlined } from '@ant-design/icons'; +import React, { useEffect, useState } from 'react'; +import styled from 'styled-components'; +import { useEntityData, useRefetch } from '../../shared/EntityContext'; +import { SidebarHeader } from '../../shared/containers/profile/sidebar/SidebarHeader'; +import { useUpdateBusinessAttributeMutation } from '../../../../graphql/businessAttribute.generated'; +import { SchemaFieldDataType } from '../../../businessAttribute/businessAttributeUtils'; + +interface Props { + readOnly?: boolean; +} + +const DataTypeSelect = styled(Select)` + && { + width: 100%; + margin-top: 1em; + margin-bottom: 1em; + } +`; +// Ensures that any newly added datatype is automatically included in the user dropdown. +const DATA_TYPES = Object.values(SchemaFieldDataType); +export const BusinessAttributeDataTypeSection = ({ readOnly }: Props) => { + const { urn, entityData } = useEntityData(); + const [originalDescription, setOriginalDescription] = useState(null); + const [isEditing, setEditing] = useState(false); + const refetch = useRefetch(); + + useEffect(() => { + if (entityData?.properties?.businessAttributeDataType) { + setOriginalDescription(entityData?.properties?.businessAttributeDataType); + } + }, [entityData]); + + const [updateBusinessAttribute] = useUpdateBusinessAttributeMutation(); + + const handleChange = (value) => { + if (value === originalDescription) { + setEditing(false); + return; + } + + updateBusinessAttribute({ variables: { urn, input: { type: value } } }) + .then(() => { + setEditing(false); + setOriginalDescription(value); + message.success({ content: 'Data Type Updated', duration: 2 }); + refetch(); + }) + .catch((e: unknown) => { + message.destroy(); + if (e instanceof Error) { + message.error({ content: `Failed to update Data Type: \n ${e.message || ''}`, duration: 3 }); + } + }); + }; + + // Toggle editing mode + const handleEditClick = () => { + setEditing(!isEditing); + }; + + return ( +
+ + + + ) + } + /> + {originalDescription} + {isEditing && ( + + {DATA_TYPES.map((dataType: SchemaFieldDataType) => ( + + {dataType} + + ))} + + )} +
+ ); +}; + +export default BusinessAttributeDataTypeSection; diff --git a/datahub-web-react/src/app/entity/businessAttribute/profile/BusinessAttributeRelatedEntity.tsx b/datahub-web-react/src/app/entity/businessAttribute/profile/BusinessAttributeRelatedEntity.tsx new file mode 100644 index 00000000000000..46d9d4ea51d245 --- /dev/null +++ b/datahub-web-react/src/app/entity/businessAttribute/profile/BusinessAttributeRelatedEntity.tsx @@ -0,0 +1,44 @@ +import * as React from 'react'; +import { UnionType } from '../../../search/utils/constants'; +import { EmbeddedListSearchSection } from '../../shared/components/styled/search/EmbeddedListSearchSection'; + +import { useEntityData } from '../../shared/EntityContext'; + +export default function BusinessAttributeRelatedEntity() { + const { entityData } = useEntityData(); + + const entityUrn = entityData?.urn; + + const fixedOrFilters = + (entityUrn && [ + { + field: 'businessAttribute', + values: [entityUrn], + }, + ]) || + []; + + entityData?.isAChildren?.relationships.forEach((businessAttribute) => { + const childUrn = businessAttribute.entity?.urn; + + if (childUrn) { + fixedOrFilters.push({ + field: 'businessAttributes', + values: [childUrn], + }); + } + }); + + return ( + + ); +} diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx index 2cd4cbd6dcb6ca..ce8d03fbdc9602 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx @@ -11,6 +11,7 @@ import SchemaEditableContext from '../../../../../shared/SchemaEditableContext'; import { useEntityData } from '../../../../shared/EntityContext'; import analytics, { EventType, EntityActionType } from '../../../../../analytics'; import { Editor } from '../../../../shared/tabs/Documentation/components/editor/Editor'; +import { ANTD_GRAY } from '../../../../shared/constants'; const EditIcon = styled(EditOutlined)` cursor: pointer; @@ -77,9 +78,25 @@ const StyledViewer = styled(Editor)` } `; +const AttributeDescription = styled.div` + margin-top: 8px; + color: ${ANTD_GRAY[7]}; +`; + +const StyledAttributeViewer = styled(Editor)` + padding-right: 8px; + display: block; + .remirror-editor.ProseMirror { + padding: 0; + color: ${ANTD_GRAY[7]}; + } +`; + type Props = { onExpanded: (expanded: boolean) => void; + onBAExpanded?: (expanded: boolean) => void; expanded: boolean; + baExpanded?: boolean; description: string; original?: string | null; onUpdate: ( @@ -87,24 +104,31 @@ type Props = { ) => Promise, Record> | void>; isEdited?: boolean; isReadOnly?: boolean; + businessAttributeDescription?: string; }; const ABBREVIATED_LIMIT = 80; export default function DescriptionField({ expanded, + baExpanded, onExpanded: handleExpanded, + onBAExpanded: handleBAExpanded, description, onUpdate, isEdited = false, original, isReadOnly, + businessAttributeDescription, }: Props) { const [showAddModal, setShowAddModal] = useState(false); const overLimit = removeMarkdown(description).length > 80; const isSchemaEditable = React.useContext(SchemaEditableContext) && !isReadOnly; const onCloseModal = () => setShowAddModal(false); const { urn, entityType } = useEntityData(); + const attributeDescriptionOverLimit = businessAttributeDescription + ? removeMarkdown(businessAttributeDescription).length > 80 + : false; const sendAnalytics = () => { analytics.event({ @@ -199,6 +223,54 @@ export default function DescriptionField({ + Add Description )} + + {baExpanded || !attributeDescriptionOverLimit ? ( + <> + {!!businessAttributeDescription && ( + + )} + {!!businessAttributeDescription && ( + + {attributeDescriptionOverLimit && ( + { + e.stopPropagation(); + if (handleBAExpanded) { + handleBAExpanded(false); + } + }} + > + Read Less + + )} + + )} + + ) : ( + <> + + { + e.stopPropagation(); + if (handleBAExpanded) { + handleBAExpanded(true); + } + }} + > + Read More + + + } + shouldWrap + > + {businessAttributeDescription} + + + )} + ); } diff --git a/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx b/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx index 7e712eef42f8de..8bbc0a693b2231 100644 --- a/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx +++ b/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx @@ -17,6 +17,7 @@ import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutS import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; import { EntityActionItem } from '../shared/entity/EntityActions'; import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection'; +import { PageRoutes } from '../../../conf/Global'; /** * Definition of the DataHub Dataset entity. @@ -59,6 +60,8 @@ export class GlossaryTermEntity implements Entity { useEntityQuery = useGetGlossaryTermQuery; + getCustomCardUrlPath = () => PageRoutes.GLOSSARY; + renderProfile = (urn) => { return ( { + type: EntityType = EntityType.SchemaField; + + icon = (fontSize: number, styleType: IconStyleType, color = '#BFBFBF') => ( + + ); + + isSearchEnabled = () => true; + + isBrowseEnabled = () => false; + + isLineageEnabled = () => false; + + // Currently unused. + getAutoCompleteFieldName = () => 'schemaField'; + + // Currently unused. + getPathName = () => 'schemaField'; + + // Currently unused. + getEntityName = () => 'schemaField'; + + // Currently unused. + getCollectionName = () => 'schemaFields'; + + // Currently unused. + renderProfile = (_: string) => <>; + + renderPreview = (previewType: PreviewType, data: SchemaFieldEntity) => ( + + ); + + renderSearch = (result: SearchResult) => this.renderPreview(PreviewType.SEARCH, result.entity as SchemaFieldEntity); + + displayName = (data: SchemaFieldEntity) => data?.fieldPath || data.urn; + + getGenericEntityProperties = (data: SchemaFieldEntity) => + getDataForEntityType({ data, entityType: this.type, getOverrideProperties: (newData) => newData }); + + supportedCapabilities = () => new Set([]); +} diff --git a/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx b/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx new file mode 100644 index 00000000000000..3f24b3a06e3a42 --- /dev/null +++ b/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx @@ -0,0 +1,38 @@ +import React from 'react'; +import { PicCenterOutlined } from '@ant-design/icons'; +import { EntityType, Owner } from '../../../../types.generated'; +import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import { IconStyleType, PreviewType } from '../../Entity'; + +export const Preview = ({ + datasetUrn, + name, + description, + owners, + previewType, +}: { + datasetUrn: string; + name: string; + description?: string | null; + owners?: Array | null; + previewType: PreviewType; +}): JSX.Element => { + const entityRegistry = useEntityRegistry(); + + const url = `${entityRegistry.getEntityUrl(EntityType.Dataset, datasetUrn)}/${encodeURIComponent('Schema')}?schemaFilter=${encodeURIComponent(name)}`; + + return ( + } + type="Column" + typeIcon={entityRegistry.getIcon(EntityType.SchemaField, 14, IconStyleType.ACCENT)} + /> + ); +}; \ No newline at end of file diff --git a/datahub-web-react/src/app/entity/shared/constants.ts b/datahub-web-react/src/app/entity/shared/constants.ts index 9df5923d185423..edf71aa608a16a 100644 --- a/datahub-web-react/src/app/entity/shared/constants.ts +++ b/datahub-web-react/src/app/entity/shared/constants.ts @@ -78,6 +78,10 @@ export const EMPTY_MESSAGES = { title: 'Is not inherited by any terms', description: 'Terms can be inherited by other terms to represent an "Is A" style relationship.', }, + businessAttributes: { + title: 'No business attributes added yet', + description: 'Add business attributes to entities to classify their data.', + }, }; export const ELASTIC_MAX_COUNT = 10000; diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx index 69389f5dcf6fc0..09fa23dbc9f57c 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx @@ -69,6 +69,8 @@ export function getCanEditName( return privileges?.manageDomains; case EntityType.DataProduct: return true; // TODO: add permissions for data products + case EntityType.BusinessAttribute: + return privileges?.manageBusinessAttributes; default: return false; } diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/AboutSection/DescriptionSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/AboutSection/DescriptionSection.tsx index a9c406306880d2..7aba714aa76e35 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/AboutSection/DescriptionSection.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/AboutSection/DescriptionSection.tsx @@ -1,11 +1,15 @@ import { Typography } from 'antd'; import React, { useState } from 'react'; import styled from 'styled-components/macro'; +import { useHistory } from 'react-router'; import CompactContext from '../../../../../../shared/CompactContext'; import MarkdownViewer, { MarkdownView } from '../../../../components/legacy/MarkdownViewer'; import NoMarkdownViewer, { removeMarkdown } from '../../../../components/styled/StripMarkdownText'; import { useRouteToTab } from '../../../../EntityContext'; import { useIsOnTab } from '../../utils'; +import { ANTD_GRAY } from '../../../../constants'; +import { EntityType } from '../../../../../../../types.generated'; +import { useEntityRegistry } from '../../../../../../useEntityRegistry'; const ABBREVIATED_LIMIT = 150; @@ -17,18 +21,35 @@ const ContentWrapper = styled.div` } `; +const BaContentWrapper = styled.div` + margin-top: 8px; + color: ${ANTD_GRAY[7]}; + margin-bottom: 8px; + font-size: 14px; + ${MarkdownView} { + font-size: 14px; + } + color: ${ANTD_GRAY[7]}; +`; + interface Props { description: string; + baDescription?: string; isExpandable?: boolean; limit?: number; + baUrn?: string; } -export default function DescriptionSection({ description, isExpandable, limit }: Props) { +export default function DescriptionSection({ description, baDescription, isExpandable, limit, baUrn }: Props) { + const history = useHistory(); const isOverLimit = description && removeMarkdown(description).length > ABBREVIATED_LIMIT; + const isBaOverLimit = baDescription && removeMarkdown(baDescription).length > ABBREVIATED_LIMIT; const [isExpanded, setIsExpanded] = useState(!isOverLimit); + const [isBaExpanded, setIsBaExpanded] = useState(!isBaOverLimit); const routeToTab = useRouteToTab(); const isCompact = React.useContext(CompactContext); const shouldShowReadMore = !useIsOnTab('Documentation') || isExpandable; + const entityRegistry = useEntityRegistry(); // if we're not in compact mode, route them to the Docs tab for the best documentation viewing experience function readMore() { @@ -39,25 +60,54 @@ export default function DescriptionSection({ description, isExpandable, limit }: } } + function readBAMore() { + if(isCompact || isExpandable) { + setIsBaExpanded(true); + } else if (baUrn != null) { + history.push(entityRegistry.getEntityUrl(EntityType.BusinessAttribute, baUrn || '')); + } + } + return ( - - {isExpanded && ( - <> - - {isOverLimit && setIsExpanded(false)}>Read Less} - - )} - {!isExpanded && ( - Read More : undefined - } - shouldWrap - > - {description} - - )} - + <> + + {isExpanded && ( + <> + + {isOverLimit && setIsExpanded(false)}>Read Less} + + )} + {!isExpanded && ( + Read More : undefined + } + shouldWrap + > + {description} + + )} + + + {isBaExpanded && ( + <> + + {isBaOverLimit && setIsBaExpanded(false)}>Read Less} + + )} + {!isBaExpanded && ( + Read More : undefined + } + shouldWrap + > + {baDescription} + + )} + + ); } diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarTagsSection.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarTagsSection.tsx index 2c0c1cfd596942..955152443ef300 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarTagsSection.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarTagsSection.tsx @@ -9,6 +9,7 @@ import { ENTITY_PROFILE_GLOSSARY_TERMS_ID, ENTITY_PROFILE_TAGS_ID, } from '../../../../../onboarding/config/EntityProfileOnboardingConfig'; +import { getNestedValue } from '../utils'; const StyledDivider = styled(Divider)` margin: 16px 0; @@ -34,7 +35,11 @@ export const SidebarTagsSection = ({ properties, readOnly }: Props) => { { (o || {})[p], obj); +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx index bd092e86b3584a..a2176b5637be86 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx @@ -25,6 +25,8 @@ import { ANTD_GRAY, ANTD_GRAY_V2 } from '../../../constants'; import translateFieldPath from '../../../../dataset/profile/schema/utils/translateFieldPath'; import PropertiesColumn from './components/PropertiesColumn'; import SchemaFieldDrawer from './components/SchemaFieldDrawer/SchemaFieldDrawer'; +import useBusinessAttributeRenderer from './utils/useBusinessAttributeRenderer'; +import { useBusinessAttributesFlag } from '../../../../../useAppConfig'; const TableContainer = styled.div` overflow: inherit; @@ -89,6 +91,7 @@ export default function SchemaTable({ hasProperties, inputFields, }: Props): JSX.Element { + const businessAttributesFlag = useBusinessAttributesFlag(); const hasUsageStats = useMemo(() => (usageStats?.aggregations?.fields?.length || 0) > 0, [usageStats]); const [tableHeight, setTableHeight] = useState(0); const [selectedFkFieldPath, setSelectedFkFieldPath] = useState , }; - let allColumns: ColumnsType = [fieldColumn, descriptionColumn, tagColumn, termColumn]; + let allColumns: ColumnsType = [ + fieldColumn, + descriptionColumn, + tagColumn, + termColumn, + ]; + + if (businessAttributesFlag) { + allColumns = [...allColumns, businessAttributeColumn]; + } if (hasProperties) { allColumns = [...allColumns, propertiesColumn]; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldAttribute.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldAttribute.tsx new file mode 100644 index 00000000000000..2df6ec5693dcd0 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldAttribute.tsx @@ -0,0 +1,31 @@ +import React from 'react'; +import { SchemaField } from '../../../../../../../../types.generated'; +import useBusinessAttributeRenderer from '../../utils/useBusinessAttributeRenderer'; +import { SectionHeader, StyledDivider } from './components'; +import SchemaEditableContext from '../../../../../../../shared/SchemaEditableContext'; +import { useBusinessAttributesFlag } from '../../../../../../../useAppConfig'; + +interface Props { + expandedField: SchemaField; +} + +export default function FieldAttribute({ expandedField }: Props) { + const isSchemaEditable = React.useContext(SchemaEditableContext); + const attributeRenderer = useBusinessAttributeRenderer( + '', + isSchemaEditable, + ); + + const businessAttributesFlag = useBusinessAttributesFlag(); + + return businessAttributesFlag ? ( + <> + Business Attribute + {/* pass in globalTags since this is a shared component, tags will not be shown or used */} +
+ {attributeRenderer('', expandedField)} +
+ + + ) : null; +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldDescription.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldDescription.tsx index 410d2801d51c87..2cd35c0f5c5b2c 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldDescription.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/components/SchemaFieldDrawer/FieldDescription.tsx @@ -77,13 +77,15 @@ export default function FieldDescription({ expandedField, editableFieldInfo }: P }); const displayedDescription = editableFieldInfo?.description || expandedField.description; + const baDescription = expandedField?.schemaFieldEntity?.businessAttributes?.businessAttribute?.businessAttribute?.properties?.description; + const baUrn = expandedField?.schemaFieldEntity?.businessAttributes?.businessAttribute?.businessAttribute?.urn; return ( <>
Description - +
{isSchemaEditable && ( - + + )} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useBusinessAttributeRenderer.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useBusinessAttributeRenderer.tsx new file mode 100644 index 00000000000000..6bedb96796d41a --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useBusinessAttributeRenderer.tsx @@ -0,0 +1,37 @@ +import React from 'react'; +import { EntityType, SchemaField } from '../../../../../../../types.generated'; +import { useRefetch } from '../../../../EntityContext'; +import { useSchemaRefetch } from '../SchemaContext'; +import BusinessAttributeGroup from '../../../../../../shared/businessAttribute/BusinessAttributeGroup'; +import { useBusinessAttributesFlag } from '../../../../../../useAppConfig'; + +export default function useBusinessAttributeRenderer( + filterText: string, + canEdit: boolean, +) { + const refetch = useRefetch(); + const schemaRefetch = useSchemaRefetch(); + + const businessAttributesFlag = useBusinessAttributesFlag(); + + const refresh: any = () => { + refetch?.(); + schemaRefetch?.(); + }; + + return (businessAttribute: string, record: SchemaField): JSX.Element | null => { + return businessAttributesFlag ? ( + + ) : null; + }; +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx index 5f2b5d23771c07..ffd288dca51bf4 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useDescriptionRenderer.tsx @@ -1,18 +1,18 @@ import React, { useState } from 'react'; import DOMPurify from 'dompurify'; -import { EditableSchemaMetadata, SchemaField, SubResourceType } from '../../../../../../../types.generated'; +import { SchemaField, SubResourceType } from '../../../../../../../types.generated'; import DescriptionField from '../../../../../dataset/profile/schema/components/SchemaDescriptionField'; -import { pathMatchesNewPath } from '../../../../../dataset/profile/schema/utils/utils'; import { useUpdateDescriptionMutation } from '../../../../../../../graphql/mutations.generated'; import { useMutationUrn, useRefetch } from '../../../../EntityContext'; import { useSchemaRefetch } from '../SchemaContext'; -export default function useDescriptionRenderer(editableSchemaMetadata: EditableSchemaMetadata | null | undefined) { +export default function useDescriptionRenderer() { const urn = useMutationUrn(); const refetch = useRefetch(); const schemaRefetch = useSchemaRefetch(); const [updateDescription] = useUpdateDescriptionMutation(); const [expandedRows, setExpandedRows] = useState({}); + const [expandedBARows, setExpandedBARows] = useState({}); const refresh: any = () => { refetch?.(); @@ -20,22 +20,26 @@ export default function useDescriptionRenderer(editableSchemaMetadata: EditableS }; return (description: string, record: SchemaField, index: number): JSX.Element => { - const relevantEditableFieldInfo = editableSchemaMetadata?.editableSchemaFieldInfo.find( - (candidateEditableFieldInfo) => pathMatchesNewPath(candidateEditableFieldInfo.fieldPath, record.fieldPath), - ); - const displayedDescription = relevantEditableFieldInfo?.description || description; + const displayedDescription = record?.description || description; const sanitizedDescription = DOMPurify.sanitize(displayedDescription); const original = record.description ? DOMPurify.sanitize(record.description) : undefined; + const businessAttributeDescription = + record?.schemaFieldEntity?.businessAttributes?.businessAttribute?.businessAttribute?.properties + ?.description || ''; const handleExpandedRows = (expanded) => setExpandedRows((prev) => ({ ...prev, [index]: expanded })); + const handleBAExpandedRows = (expanded) => setExpandedBARows((prev) => ({ ...prev, [index]: expanded })); return ( updateDescription({ variables: { diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRenderer.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRenderer.tsx index 207deb31d7ab7b..4dd11e3ee80c57 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRenderer.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/useTagsAndTermsRenderer.tsx @@ -25,11 +25,14 @@ export default function useTagsAndTermsRenderer( (candidateEditableFieldInfo) => pathMatchesNewPath(candidateEditableFieldInfo.fieldPath, record.fieldPath), ); + const businessAttributeTags = record?.schemaFieldEntity?.businessAttributes?.businessAttribute?.businessAttribute?.properties?.tags?.tags || []; + const businessAttributeTerms = record?.schemaFieldEntity?.businessAttributes?.businessAttribute?.businessAttribute?.properties?.glossaryTerms?.terms || []; + return ( ; sourceUrl?: Maybe; sourceRef?: Maybe; + businessAttributeDataType?: Maybe; }>; globalTags?: Maybe; glossaryTerms?: Maybe; diff --git a/datahub-web-react/src/app/home/HomePageRecommendations.tsx b/datahub-web-react/src/app/home/HomePageRecommendations.tsx index cc9f4b265455b2..6574b70b20de6a 100644 --- a/datahub-web-react/src/app/home/HomePageRecommendations.tsx +++ b/datahub-web-react/src/app/home/HomePageRecommendations.tsx @@ -21,6 +21,7 @@ import { HOME_PAGE_PLATFORMS_ID, } from '../onboarding/config/HomePageOnboardingConfig'; import { useToggleEducationStepIdsAllowList } from '../onboarding/useToggleEducationStepIdsAllowList'; +import { useBusinessAttributesFlag } from '../useAppConfig'; const PLATFORMS_MODULE_ID = 'Platforms'; const MOST_POPULAR_MODULE_ID = 'HighUsageEntities'; @@ -104,6 +105,8 @@ export const HomePageRecommendations = ({ user }: Props) => { const browseEntityList = entityRegistry.getBrowseEntityTypes(); const userUrn = user?.urn; + const businessAttributesFlag = useBusinessAttributesFlag(); + const showSimplifiedHomepage = user?.settings?.appearance?.showSimplifiedHomepage; const { data: entityCountData } = useGetEntityCountsQuery({ @@ -182,7 +185,22 @@ export const HomePageRecommendations = ({ user }: Props) => { {orderedEntityCounts.map( (entityCount) => entityCount && - entityCount.count !== 0 && ( + entityCount.count !== 0 && + entityCount.entityType !== EntityType.BusinessAttribute && + ( + + ), + )} + {orderedEntityCounts.map( + (entityCount) => + entityCount && + entityCount.count !== 0 && + entityCount.entityType === EntityType.BusinessAttribute && + businessAttributesFlag && ( { (entityCount) => entityCount.entityType === EntityType.GlossaryTerm, ) && } - ) : ( + ) : ( - )} + )} )} {recommendationModules && diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts index b5bbe698ed82d1..d90faa91b85a26 100644 --- a/datahub-web-react/src/app/ingest/source/builder/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts @@ -33,6 +33,7 @@ import dynamodbLogo from '../../../../images/dynamodblogo.png'; import fivetranLogo from '../../../../images/fivetranlogo.png'; import csvLogo from '../../../../images/csv-logo.png'; import qlikLogo from '../../../../images/qliklogo.png'; +import sigmaLogo from '../../../../images/sigmalogo.png'; export const ATHENA = 'athena'; export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`; @@ -119,6 +120,8 @@ export const CSV = 'csv-enricher'; export const CSV_URN = `urn:li:dataPlatform:${CSV}`; export const QLIK_SENSE = 'qlik-sense'; export const QLIK_SENSE_URN = `urn:li:dataPlatform:${QLIK_SENSE}`; +export const SIGMA = 'sigma'; +export const SIGMA_URN = `urn:li:dataPlatform:${SIGMA}`; export const PLATFORM_URN_TO_LOGO = { [ATHENA_URN]: athenaLogo, @@ -157,6 +160,7 @@ export const PLATFORM_URN_TO_LOGO = { [FIVETRAN_URN]: fivetranLogo, [CSV_URN]: csvLogo, [QLIK_SENSE_URN]: qlikLogo, + [SIGMA_URN]: sigmaLogo, }; export const SOURCE_TO_PLATFORM_URN = { diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index fa582c89d2f8ee..d4faf82a20605a 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -244,6 +244,13 @@ "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/qlik-sense/", "recipe": "source:\n type: qlik-sense\n config:\n # Coordinates\n tenant_hostname: https://xyz12xz.us.qlikcloud.com\n # Coordinates\n api_key: QLIK_API_KEY\n\n # Optional - filter for certain space names instead of ingesting everything.\n # space_pattern:\n\n # allow:\n # - space_name\n ingest_owner: true" }, + { + "urn": "urn:li:dataPlatform:sigma", + "name": "sigma", + "displayName": "Sigma", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/sigma/", + "recipe": "source:\n type: sigma\n config:\n # Coordinates\n api_url: https://aws-api.sigmacomputing.com/v2\n # Coordinates\n client_id: CLIENT_ID\n client_secret: CLIENT_SECRET\n\n # Optional - filter for certain workspace names instead of ingesting everything.\n # workspace_pattern:\n\n # allow:\n # - workspace_name\n ingest_owner: true" + }, { "urn": "urn:li:dataPlatform:cockroachdb", "name": "cockroachdb", diff --git a/datahub-web-react/src/app/search/BrowseEntityCard.tsx b/datahub-web-react/src/app/search/BrowseEntityCard.tsx index 76da58e1ed6d2e..beacce04b4340e 100644 --- a/datahub-web-react/src/app/search/BrowseEntityCard.tsx +++ b/datahub-web-react/src/app/search/BrowseEntityCard.tsx @@ -18,9 +18,9 @@ export const BrowseEntityCard = ({ entityType, count }: { entityType: EntityType const history = useHistory(); const entityRegistry = useEntityRegistry(); const showBrowseV2 = useIsBrowseV2(); - const isGlossaryEntityCard = entityType === EntityType.GlossaryTerm; const entityPathName = entityRegistry.getPathName(entityType); - const url = isGlossaryEntityCard ? PageRoutes.GLOSSARY : `${PageRoutes.BROWSE}/${entityPathName}`; + const customCardUrlPath = entityRegistry.getCustomCardUrlPath(entityType); + const url = customCardUrlPath || `${PageRoutes.BROWSE}/${entityPathName}`; const onBrowseEntityCardClick = () => { analytics.event({ type: EventType.HomePageBrowseResultClickEvent, @@ -29,7 +29,7 @@ export const BrowseEntityCard = ({ entityType, count }: { entityType: EntityType }; function browse() { - if (showBrowseV2 && !isGlossaryEntityCard) { + if (showBrowseV2 && !customCardUrlPath) { navigateToSearchUrl({ query: '*', filters: [{ field: ENTITY_SUB_TYPE_FILTER_NAME, values: [entityType] }], diff --git a/datahub-web-react/src/app/search/sidebar/useAggregationsQuery.ts b/datahub-web-react/src/app/search/sidebar/useAggregationsQuery.ts index c32dca8ba05377..bd9ac540b02204 100644 --- a/datahub-web-react/src/app/search/sidebar/useAggregationsQuery.ts +++ b/datahub-web-react/src/app/search/sidebar/useAggregationsQuery.ts @@ -51,7 +51,11 @@ const useAggregationsQuery = ({ facets, excludeFilters = false, skip }: Props) = ?.find((facet) => facet.field === ENTITY_FILTER_NAME) ?.aggregations.filter((aggregation) => { const type = aggregation.value as EntityType; - return registry.getEntity(type).isBrowseEnabled() && !GLOSSARY_ENTITY_TYPES.includes(type); + return ( + registry.getEntity(type).isBrowseEnabled() && + !GLOSSARY_ENTITY_TYPES.includes(type) && + EntityType.BusinessAttribute !== type + ); }) .sort((a, b) => { const nameA = registry.getCollectionName(a.value as EntityType); diff --git a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx index 7d53afda2aa3a6..467e535f9bad46 100644 --- a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx +++ b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx @@ -7,10 +7,11 @@ import { SettingOutlined, SolutionOutlined, DownOutlined, + GlobalOutlined, } from '@ant-design/icons'; import { Link } from 'react-router-dom'; import { Button, Dropdown, Menu, Tooltip } from 'antd'; -import { useAppConfig } from '../../useAppConfig'; +import { useAppConfig, useBusinessAttributesFlag } from '../../useAppConfig'; import { ANTD_GRAY } from '../../entity/shared/constants'; import { HOME_PAGE_INGESTION_ID } from '../../onboarding/config/HomePageOnboardingConfig'; import { useToggleEducationStepIdsAllowList } from '../../onboarding/useToggleEducationStepIdsAllowList'; @@ -66,6 +67,8 @@ export function HeaderLinks(props: Props) { const me = useUserContext(); const { config } = useAppConfig(); + const businessAttributesFlag = useBusinessAttributesFlag(); + const isAnalyticsEnabled = config?.analyticsConfig.enabled; const isIngestionEnabled = config?.managedIngestionConfig.enabled; @@ -119,6 +122,20 @@ export function HeaderLinks(props: Props) { Manage related groups of data assets + {businessAttributesFlag && ( + + + + Business Attribute + + Universal field for data consistency + + )} } > diff --git a/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx b/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx new file mode 100644 index 00000000000000..88f6a4c9660d3a --- /dev/null +++ b/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx @@ -0,0 +1,376 @@ +import React, { useRef, useState } from 'react'; +import { Button, message, Modal, Select, Tag as CustomTag } from 'antd'; +import styled from 'styled-components'; +import { GlobalOutlined } from '@ant-design/icons'; +import { Entity, EntityType, ResourceRefInput } from '../../../types.generated'; +import { useEntityRegistry } from '../../useEntityRegistry'; +import { handleBatchError } from '../../entity/shared/utils'; +import { + useAddBusinessAttributeMutation, + useRemoveBusinessAttributeMutation, +} from '../../../graphql/mutations.generated'; +import { useGetSearchResultsLazyQuery } from '../../../graphql/search.generated'; +import ClickOutside from '../ClickOutside'; +import { useGetRecommendations } from '../recommendation'; +import { useEnterKeyListener } from '../useEnterKeyListener'; +import { ENTER_KEY_CODE } from '../constants'; +import AttributeBrowser from '../../businessAttribute/AttributeBrowser'; +import { useListBusinessAttributesQuery } from '../../../graphql/businessAttribute.generated'; + +export enum OperationType { + ADD, + REMOVE, +} + +const AttributeSelect = styled(Select)` + width: 480px; +`; + +const AttributeName = styled.span` + margin-left: 5px; +`; + +const StyleTag = styled(CustomTag)` + margin: 2px; + display: flex; + justify-content: start; + align-items: center; + white-space: nowrap; + opacity: 1; + color: #434343; + line-height: 16px; + white-space: normal; + word-break: break-all; +`; + +export const BrowserWrapper = styled.div<{ isHidden: boolean; width?: string; maxHeight?: number }>` + background-color: white; + border-radius: 5px; + box-shadow: 0 3px 6px -4px rgb(0 0 0 / 12%), 0 6px 16px 0 rgb(0 0 0 / 8%), 0 9px 28px 8px rgb(0 0 0 / 5%); + max-height: ${(props) => (props.maxHeight ? props.maxHeight : '380')}px; + overflow: auto; + position: absolute; + transition: opacity 0.2s; + width: ${(props) => (props.width ? props.width : '480px')}; + z-index: 1051; + ${(props) => + props.isHidden && + ` + opacity: 0; + height: 0; + `} +`; + +type EditAttributeModalProps = { + visible: boolean; + onCloseModal: () => void; + resources: ResourceRefInput[]; + type?: EntityType; + operationType?: OperationType; + onOkOverride?: (result: string) => void; +}; + +export default function EditBusinessAttributeModal({ + visible, + type = EntityType.BusinessAttribute, + operationType = OperationType.ADD, + onCloseModal, + onOkOverride, + resources, +}: EditAttributeModalProps) { + const entityRegistry = useEntityRegistry(); + const [inputValue, setInputValue] = useState(''); + const [addBusinessAttributeMutation] = useAddBusinessAttributeMutation(); + const [removeBusinessAttributeMutation] = useRemoveBusinessAttributeMutation(); + const [isFocusedOnInput, setIsFocusedOnInput] = useState(false); + const inputEl = useRef(null); + const [urn, setUrn] = useState(''); + const [disableAction, setDisableAction] = useState(false); + const [recommendedData] = useGetRecommendations([EntityType.BusinessAttribute]); + const [selectedAttribute, setSelectedAttribute] = useState(''); + const [attributeSearch, { data: attributeSearchData }] = useGetSearchResultsLazyQuery(); + const attributeSearchResults = + attributeSearchData?.search?.searchResults?.map((searchResult) => searchResult.entity) || []; + const { data: attributeData } = useListBusinessAttributesQuery({ + variables: { + start: 0, + count: 10, + query: '', + }, + }); + + const displayedAttributes = + attributeData?.listBusinessAttributes?.businessAttributes?.map((defaultValue) => ({ + urn: defaultValue.urn, + component: ( +
+ + {defaultValue?.properties?.name} +
+ ), + })) || []; + + const handleSearch = (text: string) => { + if (text.length > 0) { + attributeSearch({ + variables: { + input: { + type, + query: text, + start: 0, + count: 10, + }, + }, + }); + } + }; + + const renderSearchResult = (entity: Entity) => { + const displayName = entityRegistry.getDisplayName(entity.type, entity); + return ( + +
+ + {displayName} +
+ + ); + }; + + const attributeResult = !inputValue || inputValue.length === 0 ? recommendedData : attributeSearchResults; + const attributeSearchOptions = attributeResult?.map((result) => { + return renderSearchResult(result); + }); + + const attributeRender = (props) => { + // eslint-disable-next-line react/prop-types + const { closable, onClose, value } = props; + const onPreventMouseDown = (event) => { + event.preventDefault(); + event.stopPropagation(); + }; + /* eslint-disable-next-line react/prop-types */ + const selectedItem = displayedAttributes.find((attribute) => attribute.urn === value)?.component; + return ( + + {selectedItem} + + ); + }; + + // Handle the Enter press + useEnterKeyListener({ + querySelectorToExecuteClick: '#addAttributeButton', + }); + + // When business attribute search result is selected, add the urn + const onSelectValue = (selectedUrn: string) => { + const selectedSearchOption = attributeSearchOptions?.find((option) => option.props.value === selectedUrn); + setUrn(selectedUrn); + if (!selectedAttribute) { + setSelectedAttribute({ + selectedUrn, + component: ( +
+ + {selectedSearchOption?.props.name} +
+ ), + }); + } + if (inputEl && inputEl.current) { + (inputEl.current as any).blur(); + } + }; + + // When a Tag or term search result is deselected, remove the urn from the Owners + const onDeselectValue = (selectedUrn: string) => { + setUrn(urn === selectedUrn ? '' : urn); + setInputValue(''); + setIsFocusedOnInput(true); + setSelectedAttribute(''); + }; + + const addBusinessAttribute = () => { + addBusinessAttributeMutation({ + variables: { + input: { + businessAttributeUrn: urn, + resourceUrn: resources, + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + message.success({ + content: `Added Business Attribute!`, + duration: 2, + }); + } + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to add: \n ${e.message || ''}`, duration: 3 }); + }) + .finally(() => { + setDisableAction(false); + onCloseModal(); + setUrn(''); + }); + }; + + const removeBusinessAttribute = () => { + removeBusinessAttributeMutation({ + variables: { + input: { + businessAttributeUrn: urn, + resourceUrn: [{ + resourceUrn: resources[0].resourceUrn, + subResource: resources[0].subResource, + subResourceType: resources[0].subResourceType, + }], + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + message.success({ + content: `Removed Business Attribute!`, + duration: 2, + }); + } + }) + .catch((e) => { + message.destroy(); + message.error( + handleBatchError(urn, e, { content: `Failed to remove: \n ${e.message || ''}`, duration: 3 }), + ); + }) + .finally(() => { + setDisableAction(false); + onCloseModal(); + setUrn(''); + }); + }; + + const editBusinessAttribute = () => { + if (operationType === OperationType.ADD) { + addBusinessAttribute(); + } else { + removeBusinessAttribute(); + } + }; + + const onOk = () => { + if (onOkOverride) { + onOkOverride(urn); + return; + } + + if (!resources) { + onCloseModal(); + return; + } + setDisableAction(true); + editBusinessAttribute(); + }; + + function selectAttributeFromBrowser(selectedUrn: string, displayName: string) { + setIsFocusedOnInput(false); + setUrn(selectedUrn); + setSelectedAttribute({ + selectedUrn, + component: ( +
+ + {displayName} +
+ ), + }); + } + + function clearInput() { + setInputValue(''); + setTimeout(() => setIsFocusedOnInput(true), 0); // call after click outside + } + + function handleBlur() { + setInputValue(''); + } + + function handleKeyDown(event) { + if (event.keyCode === ENTER_KEY_CODE) { + (inputEl.current as any).blur(); + } + } + + const isShowingAttributeBrowser = !inputValue && type === EntityType.BusinessAttribute && isFocusedOnInput; + + return ( + + + + + } + > + setIsFocusedOnInput(false)}> + { + onSelectValue(asset); + }} + onDeselect={(asset: any) => onDeselectValue(asset)} + onSearch={(value: string) => { + // eslint-disable-next-line react/prop-types + handleSearch(value.trim()); + // eslint-disable-next-line react/prop-types + setInputValue(value.trim()); + }} + tagRender={attributeRender} + value={urn || undefined} + onClear={clearInput} + onFocus={() => setIsFocusedOnInput(true)} + onBlur={handleBlur} + onInputKeyDown={handleKeyDown} + dropdownStyle={isShowingAttributeBrowser ? { display: 'none', color: 'RED' } : {}} + > + {attributeSearchOptions} + + + + + + + ); +} diff --git a/datahub-web-react/src/app/shared/businessAttribute/AttributeContent.tsx b/datahub-web-react/src/app/shared/businessAttribute/AttributeContent.tsx new file mode 100644 index 00000000000000..61306c9cf64d3a --- /dev/null +++ b/datahub-web-react/src/app/shared/businessAttribute/AttributeContent.tsx @@ -0,0 +1,117 @@ +import styled from 'styled-components'; +import { message, Modal, Tag } from 'antd'; +import { GlobalOutlined } from '@ant-design/icons'; +import React from 'react'; +import Highlight from 'react-highlighter'; +import { useEntityRegistry } from '../../useEntityRegistry'; +import { BusinessAttributeAssociation, EntityType } from '../../../types.generated'; +import { useHasMatchedFieldByUrn } from '../../search/context/SearchResultContext'; +import { MatchedFieldName } from '../../search/matches/constants'; +import { useRemoveBusinessAttributeMutation } from '../../../graphql/mutations.generated'; + +const highlightMatchStyle = { background: '#ffe58f', padding: '0' }; + +const StyledAttribute = styled(Tag)<{ fontSize?: number; highlightAttribute?: boolean }>` + &&& { + ${(props) => + props.highlightAttribute && + `background: ${props.theme.styles['highlight-color']}; + border: 1px solid ${props.theme.styles['highlight-border-color']}; + `} + } + ${(props) => props.fontSize && `font-size: ${props.fontSize}px;`} +`; + +interface Props { + businessAttribute: BusinessAttributeAssociation | undefined; + entityUrn?: string; + canRemove?: boolean; + readOnly?: boolean; + highlightText?: string; + fontSize?: number; + onOpenModal?: () => void; + refetch?: () => Promise; +} + +export default function AttributeContent({ + businessAttribute, + canRemove, + readOnly, + highlightText, + fontSize, + onOpenModal, + entityUrn, + refetch, +}: Props) { + const entityRegistry = useEntityRegistry(); + const [removeBusinessAttributeMutation] = useRemoveBusinessAttributeMutation(); + const highlightAttribute = useHasMatchedFieldByUrn( + businessAttribute?.businessAttribute?.urn || '', + 'businessAttributes' as MatchedFieldName, + ); + + const removeAttribute = (attributeToRemove: BusinessAttributeAssociation) => { + onOpenModal?.(); + const AttributeName = + attributeToRemove && + entityRegistry.getDisplayName( + attributeToRemove.businessAttribute.type, + attributeToRemove.businessAttribute, + ); + Modal.confirm({ + title: `Do you want to remove ${AttributeName} attribute?`, + content: `Are you sure you want to remove the ${AttributeName} attribute?`, + onOk() { + if (attributeToRemove.associatedUrn || entityUrn) { + removeBusinessAttributeMutation({ + variables: { + input: { + businessAttributeUrn: attributeToRemove.businessAttribute.urn, + resourceUrn: [{ + resourceUrn: attributeToRemove.associatedUrn || entityUrn || '', + subResource: null, + subResourceType: null, + }], + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + message.success({ content: 'Removed Business Attribute!', duration: 2 }); + } + }) + .then(refetch) + .catch((e) => { + message.destroy(); + message.error({ + content: `Failed to remove business attribute: \n ${e.message || ''}`, + duration: 3, + }); + }); + } + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + }; + + return ( + { + e.preventDefault(); + removeAttribute(businessAttribute as BusinessAttributeAssociation); + }} + fontSize={fontSize} + highlightAttribute={highlightAttribute} + > + + + {entityRegistry.getDisplayName(EntityType.BusinessAttribute, businessAttribute?.businessAttribute)} + + + ); +} diff --git a/datahub-web-react/src/app/shared/businessAttribute/BusinessAttributeGroup.tsx b/datahub-web-react/src/app/shared/businessAttribute/BusinessAttributeGroup.tsx new file mode 100644 index 00000000000000..0e77d650a14332 --- /dev/null +++ b/datahub-web-react/src/app/shared/businessAttribute/BusinessAttributeGroup.tsx @@ -0,0 +1,104 @@ +import styled from 'styled-components'; +import { Button, Typography } from 'antd'; +import React, { useState } from 'react'; +import { PlusOutlined } from '@ant-design/icons'; +import { EMPTY_MESSAGES } from '../../entity/shared/constants'; +import { BusinessAttributeAssociation, EntityType } from '../../../types.generated'; +import EditBusinessAttributeModal from './AddBusinessAttributeModal'; +import StyledAttribute from './StyledAttribute'; + +type Props = { + businessAttribute?: BusinessAttributeAssociation; + canRemove?: boolean; + canAddAttribute?: boolean; + showEmptyMessage?: boolean; + buttonProps?: Record; + onOpenModal?: () => void; + maxShow?: number; + entityUrn?: string; + entityType?: EntityType; + entitySubresource?: string; + highlightText?: string; + fontSize?: number; + refetch?: () => Promise; + readOnly?: boolean; +}; + +const NoElementButton = styled(Button)` + :not(:last-child) { + margin-right: 8px; + } +`; + +export default function BusinessAttributeGroup({ + businessAttribute, + canAddAttribute, + showEmptyMessage, + buttonProps, + onOpenModal, + entityUrn, + entityType, + entitySubresource, + refetch, + readOnly, + canRemove, + highlightText, + fontSize, +}: Props) { + const [showAddModal, setShowAddModal] = useState(false); + const [addModalType, setAddModalType] = useState(EntityType.BusinessAttribute); + const businessAttributeEmpty = !businessAttribute?.associatedUrn?.length; + return ( + <> + {!businessAttributeEmpty && businessAttribute !== undefined && ( + + )} + {showEmptyMessage && canAddAttribute && businessAttributeEmpty && ( + + {EMPTY_MESSAGES.businessAttributes.title}. {EMPTY_MESSAGES.businessAttributes.description} + + )} + {canAddAttribute && !readOnly && businessAttributeEmpty && ( + { + setAddModalType(EntityType.BusinessAttribute); + setShowAddModal(true); + }} + {...buttonProps} + > + + Add Attribute + + )} + {showAddModal && !!entityUrn && !!entityType && ( + { + onOpenModal?.(); + setShowAddModal(false); + refetch?.(); + }} + resources={[ + { + resourceUrn: entityUrn, + subResource: null, + subResourceType: null, + }, + ]} + /> + )} + + ); +} diff --git a/datahub-web-react/src/app/shared/businessAttribute/StyledAttribute.tsx b/datahub-web-react/src/app/shared/businessAttribute/StyledAttribute.tsx new file mode 100644 index 00000000000000..1a69ed23b2f007 --- /dev/null +++ b/datahub-web-react/src/app/shared/businessAttribute/StyledAttribute.tsx @@ -0,0 +1,58 @@ +import React from 'react'; +import { Link } from 'react-router-dom'; +import styled from 'styled-components'; +import { BusinessAttributeAssociation, EntityType } from '../../../types.generated'; +import { useEntityRegistry } from '../../useEntityRegistry'; +import { HoverEntityTooltip } from '../../recommendations/renderer/component/HoverEntityTooltip'; +import AttributeContent from './AttributeContent'; + +const AttributeLink = styled(Link)` + display: inline-block; + margin-bottom: 8px; +`; + +const AttributeWrapper = styled.span` + display: inline-block; + margin-bottom: 8px; +`; + +interface Props { + businessAttribute: BusinessAttributeAssociation; + entityUrn?: string; + entitySubresource?: string; + canRemove?: boolean; + readOnly?: boolean; + highlightText?: string; + fontSize?: number; + onOpenModal?: () => void; + refetch?: () => Promise; +} + +export default function StyledAttribute(props: Props) { + const { businessAttribute, readOnly } = props; + const entityRegistry = useEntityRegistry(); + + if (readOnly) { + return ( + + + + + + ); + } + + return ( + + + + + + ); +} diff --git a/datahub-web-react/src/app/shared/deleteUtils.ts b/datahub-web-react/src/app/shared/deleteUtils.ts index 37a3758712ad6c..a831f9338a53f5 100644 --- a/datahub-web-react/src/app/shared/deleteUtils.ts +++ b/datahub-web-react/src/app/shared/deleteUtils.ts @@ -3,6 +3,7 @@ import { useDeleteAssertionMutation } from '../../graphql/assertion.generated'; import { useDeleteDataProductMutation } from '../../graphql/dataProduct.generated'; import { useDeleteDomainMutation } from '../../graphql/domain.generated'; import { useDeleteGlossaryEntityMutation } from '../../graphql/glossary.generated'; +import { useDeleteBusinessAttributeMutation } from '../../graphql/businessAttribute.generated'; import { useRemoveGroupMutation } from '../../graphql/group.generated'; import { useDeleteTagMutation } from '../../graphql/tag.generated'; import { useRemoveUserMutation } from '../../graphql/user.generated'; @@ -34,6 +35,8 @@ export const getEntityProfileDeleteRedirectPath = (type: EntityType, entityData: return `/domain/${domain.urn}/Data Products`; } return '/'; + case EntityType.BusinessAttribute: + return `${PageRoutes.BUSINESS_ATTRIBUTE}`; default: return () => undefined; } @@ -63,6 +66,8 @@ export const getDeleteEntityMutation = (type: EntityType) => { return useDeleteGlossaryEntityMutation; case EntityType.DataProduct: return useDeleteDataProductMutation; + case EntityType.BusinessAttribute: + return useDeleteBusinessAttributeMutation; default: return () => undefined; } diff --git a/datahub-web-react/src/app/useAppConfig.ts b/datahub-web-react/src/app/useAppConfig.ts index 821d00b9017c31..f167ccad16474d 100644 --- a/datahub-web-react/src/app/useAppConfig.ts +++ b/datahub-web-react/src/app/useAppConfig.ts @@ -17,3 +17,13 @@ export function useIsNestedDomainsEnabled() { const appConfig = useAppConfig(); return appConfig.config.featureFlags.nestedDomainsEnabled; } + +export function useBusinessAttributesFlag() { + const appConfig = useAppConfig(); + return appConfig.config.featureFlags.businessAttributeEntityEnabled; +} + +export function useIsAppConfigContextLoaded() { + const appConfig = useAppConfig(); + return appConfig.loaded; +} diff --git a/datahub-web-react/src/appConfigContext.tsx b/datahub-web-react/src/appConfigContext.tsx index 00feaf82234100..b4f16e2d2a8240 100644 --- a/datahub-web-react/src/appConfigContext.tsx +++ b/datahub-web-react/src/appConfigContext.tsx @@ -52,6 +52,7 @@ export const DEFAULT_APP_CONFIG = { showAccessManagement: false, nestedDomainsEnabled: true, platformBrowseV2: false, + businessAttributeEntityEnabled: false, }, }; diff --git a/datahub-web-react/src/conf/Global.ts b/datahub-web-react/src/conf/Global.ts index 3da6585bfa5255..433b0b6416e780 100644 --- a/datahub-web-react/src/conf/Global.ts +++ b/datahub-web-react/src/conf/Global.ts @@ -29,6 +29,7 @@ export enum PageRoutes { EMBED = '/embed', EMBED_LOOKUP = '/embed/lookup/:url', SETTINGS_POSTS = '/settings/posts', + BUSINESS_ATTRIBUTE = '/business-attribute', } /** diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql index b7527d53b5705f..7b47fc0302247b 100644 --- a/datahub-web-react/src/graphql/app.graphql +++ b/datahub-web-react/src/graphql/app.graphql @@ -67,6 +67,7 @@ query appConfig { showAccessManagement nestedDomainsEnabled platformBrowseV2 + businessAttributeEntityEnabled } } } diff --git a/datahub-web-react/src/graphql/businessAttribute.graphql b/datahub-web-react/src/graphql/businessAttribute.graphql new file mode 100644 index 00000000000000..544a5083d1f2bc --- /dev/null +++ b/datahub-web-react/src/graphql/businessAttribute.graphql @@ -0,0 +1,84 @@ +# Get a business attribute by URN +query getBusinessAttribute($urn: String!) { + businessAttribute(urn: $urn) { + ...businessAttributeFields + } +} + +query listBusinessAttributes($start: Int!, $count: Int!, $query: String) { + listBusinessAttributes(input: { start: $start, count: $count, query: $query }) { + start + count + total + businessAttributes { + ...businessAttributeFields + } + } +} + +fragment businessAttributeFields on BusinessAttribute { + urn + type + ownership { + ...ownershipFields + } + properties { + name + description + businessAttributeDataType: type + customProperties { + key + value + associatedUrn + } + lastModified { + time + } + created { + time + } + tags { + tags { + tag { + urn + name + properties { + name + } + } + associatedUrn + } + } + glossaryTerms { + terms { + term { + urn + type + properties { + name + } + } + associatedUrn + } + } + } + institutionalMemory { + ...institutionalMemoryFields + } +} + +mutation createBusinessAttribute($input: CreateBusinessAttributeInput!) { + createBusinessAttribute(input: $input) { + ...businessAttributeFields + } +} + +mutation deleteBusinessAttribute($urn: String!) { + deleteBusinessAttribute(urn: $urn) +} + +mutation updateBusinessAttribute($urn: String!, $input: UpdateBusinessAttributeInput!) { + updateBusinessAttribute(urn: $urn, input: $input) { + ...businessAttributeFields + } +} diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index be3b2e86209716..7028ac8c4f4d0f 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -731,6 +731,11 @@ fragment schemaFieldFields on SchemaField { ...structuredPropertiesFields } } + businessAttributes { + businessAttribute { + ...businessAttribute + } + } } } @@ -1407,3 +1412,49 @@ fragment entityPrivileges on EntityPrivileges { canManageChildren canEditProperties } + +fragment businessAttribute on BusinessAttributeAssociation { + businessAttribute { + urn + type + ownership { + ...ownershipFields + } + properties { + name + description + businessAttributeDataType: type + lastModified { + time + } + created { + time + } + tags { + tags { + tag { + urn + name + properties { + name + } + } + associatedUrn + } + } + glossaryTerms { + terms { + term { + urn + type + properties { + name + } + } + associatedUrn + } + } + } + } + associatedUrn +} diff --git a/datahub-web-react/src/graphql/me.graphql b/datahub-web-react/src/graphql/me.graphql index 4f16a055a04243..7a2c0e562be6bb 100644 --- a/datahub-web-react/src/graphql/me.graphql +++ b/datahub-web-react/src/graphql/me.graphql @@ -48,6 +48,9 @@ query getMe { manageGlobalViews manageOwnershipTypes manageGlobalAnnouncements + createBusinessAttributes + manageBusinessAttributes + } } } diff --git a/datahub-web-react/src/graphql/mutations.graphql b/datahub-web-react/src/graphql/mutations.graphql index 70ec8beaa4147d..adc78d684a86de 100644 --- a/datahub-web-react/src/graphql/mutations.graphql +++ b/datahub-web-react/src/graphql/mutations.graphql @@ -139,3 +139,11 @@ mutation updateLineage($input: UpdateLineageInput!) { mutation updateEmbed($input: UpdateEmbedInput!) { updateEmbed(input: $input) } + +mutation addBusinessAttribute($input: AddBusinessAttributeInput!) { + addBusinessAttribute(input: $input) +} + +mutation removeBusinessAttribute($input: AddBusinessAttributeInput!) { + removeBusinessAttribute(input: $input) +} diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index e9cca5e64dc66d..aff506779094f3 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -241,6 +241,12 @@ fragment autoCompleteFields on Entity { ... on DataPlatform { ...nonConflictingPlatformFields } + ... on BusinessAttribute { + properties { + name + description + } + } } query getAutoCompleteResults($input: AutoCompleteInput!) { @@ -845,6 +851,12 @@ fragment searchResultFields on Entity { ...glossaryTerms } } + ... on BusinessAttribute { + ...businessAttributeFields + } + ... on SchemaFieldEntity { + ...entityField + } } fragment facetFields on FacetMetadata { @@ -952,6 +964,26 @@ fragment searchResults on SearchResults { } } +fragment entityField on SchemaFieldEntity { + urn + type + parent { + urn + type + } + fieldPath + structuredProperties { + properties { + ...structuredPropertiesFields + } + } + businessAttributes { + businessAttribute { + ...businessAttribute + } + } +} + fragment schemaFieldEntityFields on SchemaFieldEntity { urn type diff --git a/datahub-web-react/src/images/sigmalogo.png b/datahub-web-react/src/images/sigmalogo.png new file mode 100644 index 00000000000000..018c4b16745aef Binary files /dev/null and b/datahub-web-react/src/images/sigmalogo.png differ diff --git a/datahub-web-react/src/utils/test-utils/TestPageContainer.tsx b/datahub-web-react/src/utils/test-utils/TestPageContainer.tsx index 20fd5afe593b52..c07505d7315a13 100644 --- a/datahub-web-react/src/utils/test-utils/TestPageContainer.tsx +++ b/datahub-web-react/src/utils/test-utils/TestPageContainer.tsx @@ -25,6 +25,7 @@ import UserContextProvider from '../../app/context/UserContextProvider'; import { DataPlatformEntity } from '../../app/entity/dataPlatform/DataPlatformEntity'; import { ContainerEntity } from '../../app/entity/container/ContainerEntity'; import AppConfigProvider from '../../AppConfigProvider'; +import {BusinessAttributeEntity} from "../../app/entity/businessAttribute/BusinessAttributeEntity"; type Props = { children: React.ReactNode; @@ -47,6 +48,7 @@ export function getTestEntityRegistry() { entityRegistry.register(new MLModelGroupEntity()); entityRegistry.register(new DataPlatformEntity()); entityRegistry.register(new ContainerEntity()); + entityRegistry.register(new BusinessAttributeEntity()); return entityRegistry; } diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index fe1b979ab332b9..8b0bb3ea05df44 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -58,6 +58,7 @@ module.exports = { "docs/sync-status", "docs/incidents/incidents", "docs/generated/lineage/lineage-feature-guide", + "docs/businessattributes", { type: "doc", id: "docs/tests/metadata-tests", diff --git a/docs/businessattributes.md b/docs/businessattributes.md new file mode 100644 index 00000000000000..80d47f0966aae8 --- /dev/null +++ b/docs/businessattributes.md @@ -0,0 +1,80 @@ +# Business Attributes + + +## What are Business Attributes +A Business Attribute, as its name implies, is an attribute with a business focus. It embodies the traits or properties of an entity within a business framework. This attribute is a crucial piece of data for a business, utilised to define or control the entity throughout the organisation. If a business process or concept is depicted as a comprehensive logical model, then each Business Attribute can be considered as an individual component within that model. While business names and descriptions are generally managed through glossary terms, Business Attributes encompass additional characteristics such as data quality rules/assertions, data privacy markers, data usage protocols, standard tags, and supplementary documentation, alongside Names and Descriptions. + +For instance, "United States - Social Security Number" comes with a Name and definition. However, it also includes an abbreviation, a Personal Identifiable Information (PII) classification tag, a set of data rules, and possibly some additional references. + +## Benefits of Business Attributes +The principle of "Define Once; Use in Many Places" applies to Business Attributes. Information Stewards can establish these attributes once with all their associated characteristics in an enterprise environment. Subsequently, individual applications or data owners can link their dataset attributes with these Business Attributes. This process allows the complete metadata structure built for a Business Attribute to be inherited. Application owners can also use these attributes to check if their applications align with the organisation-wide standard descriptions and data policies. This approach aids in centralised management for enhanced control and enforcement of metadata standards. + +This standardised metadata can be employed to facilitate data quality, data governance, and data discovery use cases within the organisation. + +A collection of 'related' Business Attributes can create a logical business model. + +With Business Attributes users have the ability to search associated datasets using business description/tags/glossary attached to business attribute +## How can you use Business Attributes +Business Attributes can be utilised in any of the following scenario: +Attributes that are frequently used across multiple domains, data products, projects, and applications. +Attributes requiring standardisation and inheritance of their characteristics, including name and descriptions, to be propagated. +Attributes that need centralised management for improved control and standard enforcement. + +A Business Attribute could be used to accelerate and standardise business definition management at entity / fields a field across various datasets. This ensures consistent application of the characteristics across all datasets using the Business Attribute. Any change in the them requires a change at only one place (i.e., business attributes) and change can then be inherited across all the application & datasets in the organisation + +Taking the example of "United States- Social Security Number", if an application or data owner has multiple instances of the social security number within their datasets, they can link all these dataset attributes with a Business Attribute to inherit all the aforementioned characteristics. Additionally, users can search for associated datasets using the business description, tags, or glossary linked to the Business Attribute. + +## Business Attributes Setup, Prerequisites, and Permissions +What you need to create/update and associate business attributes to dataset schema field + +* **Manage Business Attributes** platform privilege to create/update/delete business attributes. +* **Edit Dataset Column Business Attribute** metadata privilege to associate business attributes to dataset schema field. + +## Using Business Attributes +As of now Business Attributes can only be created through UI + +### Creating a Business Attribute (UI) +To create a Business Attribute, first navigate to the Business Attributes tab on the home page. + +

+ +

+ +Then click on '+ Create Business Attribute'. +This will open a new modal where you can configure the settings for your business attribute. Inside the form, you can choose a name for Business Attribute. Most often, this will align with the logical purpose of the Business Attribute, +for example 'Social Security Number'. You can also add documentation for your Business Attribute to help other users easily discover it. This can be changed later. + +We can also add datatype for Business Attribute. It has String as a default value. + +

+ +

+ +Once you've chosen a name and a description, click 'Create' to create the new Business Attribute. + +Then we can attach tags and glossary terms to it to make it more discoverable. + +### Assigning Business Attribute to a Dataset Schema Field (UI) +You can associate the business attribute to a dataset schema field using the Dataset's schema page as the starting point. As per design, there is one to one mapping between business attribute and dataset schema field. + +On a Dataset's schema page, click the 'Add Attribute' to add business attribute to the dataset schema field. + +

+ +

+ +After association, dataset schema field gets its description, tags and glossary inherited from Business attribute. +Description inherited from business attribute is greyed out to differentiate between original description of schema field. Similarly, tags and glossary terms inherited can't be removed directly. + +

+ +

+ +### What updates are planned for the Business Attributes feature? + +- Ingestion of Business attributes through recipe file (YAML) +- AutoTagging of Business attributes to child datasets through lineage + +### Related Features +* [Glossary Terms](./glossary/business-glossary.md) +* [Tags](./tags.md) \ No newline at end of file diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/AspectSpec.java b/entity-registry/src/main/java/com/linkedin/metadata/models/AspectSpec.java index 9cf8b4174ecfbd..a2ff81da564017 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/AspectSpec.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/AspectSpec.java @@ -23,6 +23,7 @@ public class AspectSpec { private final Map _relationshipFieldSpecs; private final Map _timeseriesFieldSpecs; private final Map _timeseriesFieldCollectionSpecs; + private final Map _searchableRefFieldSpecs; // Classpath & Pegasus-specific: Temporary. private final RecordDataSchema _schema; @@ -37,6 +38,7 @@ public AspectSpec( @Nonnull final List relationshipFieldSpecs, @Nonnull final List timeseriesFieldSpecs, @Nonnull final List timeseriesFieldCollectionSpecs, + @Nonnull final List searchableRefFieldSpecs, final RecordDataSchema schema, final Class aspectClass) { _aspectAnnotation = aspectAnnotation; @@ -45,6 +47,11 @@ public AspectSpec( .collect( Collectors.toMap( spec -> spec.getPath().toString(), spec -> spec, (val1, val2) -> val1)); + _searchableRefFieldSpecs = + searchableRefFieldSpecs.stream() + .collect( + Collectors.toMap( + spec -> spec.getPath().toString(), spec -> spec, (val1, val2) -> val1)); _searchScoreFieldSpecs = searchScoreFieldSpecs.stream() .collect( @@ -113,6 +120,10 @@ public List getSearchableFieldSpecs() { return new ArrayList<>(_searchableFieldSpecs.values()); } + public List getSearchableRefFieldSpecs() { + return new ArrayList<>(_searchableRefFieldSpecs.values()); + } + public List getSearchScoreFieldSpecs() { return new ArrayList<>(_searchScoreFieldSpecs.values()); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/DefaultEntitySpec.java b/entity-registry/src/main/java/com/linkedin/metadata/models/DefaultEntitySpec.java index 2546674f9835cb..38a3cbdeb458a8 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/DefaultEntitySpec.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/DefaultEntitySpec.java @@ -4,11 +4,7 @@ import com.linkedin.data.schema.TyperefDataSchema; import com.linkedin.metadata.models.annotation.EntityAnnotation; import com.linkedin.metadata.models.annotation.SearchableAnnotation; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -27,6 +23,7 @@ public class DefaultEntitySpec implements EntitySpec { private List _searchableFieldSpecs; private Map> searchableFieldTypeMap; + private List _searchableRefFieldSpecs; public DefaultEntitySpec( @Nonnull final Collection aspectSpecs, @@ -106,6 +103,14 @@ public List getSearchableFieldSpecs() { return _searchableFieldSpecs; } + @Override + public List getSearchableRefFieldSpecs() { + if (_searchableRefFieldSpecs == null) { + _searchableRefFieldSpecs = EntitySpec.super.getSearchableRefFieldSpecs(); + } + return _searchableRefFieldSpecs; + } + @Override public Map> getSearchableFieldTypes() { if (searchableFieldTypeMap == null) { diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/EntitySpec.java b/entity-registry/src/main/java/com/linkedin/metadata/models/EntitySpec.java index 9a75cc1f751d3b..02fd1b22b52d6e 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/EntitySpec.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/EntitySpec.java @@ -89,4 +89,11 @@ default List getRelationshipFieldSpecs() { .flatMap(List::stream) .collect(Collectors.toList()); } + + default List getSearchableRefFieldSpecs() { + return getAspectSpecs().stream() + .map(AspectSpec::getSearchableRefFieldSpecs) + .flatMap(List::stream) + .collect(Collectors.toList()); + } } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/EntitySpecBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/models/EntitySpecBuilder.java index 54f2206798da0d..c79ea5de69e277 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/EntitySpecBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/EntitySpecBuilder.java @@ -17,6 +17,7 @@ import com.linkedin.metadata.models.annotation.RelationshipAnnotation; import com.linkedin.metadata.models.annotation.SearchScoreAnnotation; import com.linkedin.metadata.models.annotation.SearchableAnnotation; +import com.linkedin.metadata.models.annotation.SearchableRefAnnotation; import com.linkedin.metadata.models.annotation.TimeseriesFieldAnnotation; import com.linkedin.metadata.models.annotation.TimeseriesFieldCollectionAnnotation; import java.util.ArrayList; @@ -39,6 +40,8 @@ public class EntitySpecBuilder { new PegasusSchemaAnnotationHandlerImpl(SearchableAnnotation.ANNOTATION_NAME); public static SchemaAnnotationHandler _searchScoreHandler = new PegasusSchemaAnnotationHandlerImpl(SearchScoreAnnotation.ANNOTATION_NAME); + public static SchemaAnnotationHandler _searchRefScoreHandler = + new PegasusSchemaAnnotationHandlerImpl(SearchableRefAnnotation.ANNOTATION_NAME); public static SchemaAnnotationHandler _relationshipHandler = new PegasusSchemaAnnotationHandlerImpl(RelationshipAnnotation.ANNOTATION_NAME); public static SchemaAnnotationHandler _timeseriesFiledAnnotationHandler = @@ -222,6 +225,7 @@ public AspectSpec buildAspectSpec( Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), aspectRecordSchema, aspectClass); } @@ -245,6 +249,19 @@ public AspectSpec buildAspectSpec( aspectRecordSchema, new SchemaAnnotationProcessor.AnnotationProcessOption()); + final SchemaAnnotationProcessor.SchemaAnnotationProcessResult processedSearchRefResult = + SchemaAnnotationProcessor.process( + Collections.singletonList(_searchRefScoreHandler), + aspectRecordSchema, + new SchemaAnnotationProcessor.AnnotationProcessOption()); + + // Extract SearchableRef Field Specs + final SearchableRefFieldSpecExtractor searchableRefFieldSpecExtractor = + new SearchableRefFieldSpecExtractor(); + final DataSchemaRichContextTraverser searchableRefFieldSpecTraverser = + new DataSchemaRichContextTraverser(searchableRefFieldSpecExtractor); + searchableRefFieldSpecTraverser.traverse(processedSearchRefResult.getResultSchema()); + // Extract SearchScore Field Specs final SearchScoreFieldSpecExtractor searchScoreFieldSpecExtractor = new SearchScoreFieldSpecExtractor(); @@ -289,6 +306,7 @@ public AspectSpec buildAspectSpec( relationshipFieldSpecExtractor.getSpecs(), timeseriesFieldSpecExtractor.getTimeseriesFieldSpecs(), timeseriesFieldSpecExtractor.getTimeseriesFieldCollectionSpecs(), + searchableRefFieldSpecExtractor.getSpecs(), aspectRecordSchema, aspectClass); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableRefFieldSpec.java b/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableRefFieldSpec.java new file mode 100644 index 00000000000000..d4093b27cb9391 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableRefFieldSpec.java @@ -0,0 +1,19 @@ +package com.linkedin.metadata.models; + +import com.linkedin.data.schema.DataSchema; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.metadata.models.annotation.SearchableRefAnnotation; +import lombok.NonNull; +import lombok.Value; + +@Value +public class SearchableRefFieldSpec implements FieldSpec { + + @NonNull PathSpec path; + @NonNull SearchableRefAnnotation searchableRefAnnotation; + @NonNull DataSchema pegasusSchema; + + public boolean isArray() { + return path.getPathComponents().contains("*"); + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableRefFieldSpecExtractor.java b/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableRefFieldSpecExtractor.java new file mode 100644 index 00000000000000..4f03df973467a9 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableRefFieldSpecExtractor.java @@ -0,0 +1,187 @@ +package com.linkedin.metadata.models; + +import com.linkedin.data.schema.ComplexDataSchema; +import com.linkedin.data.schema.DataSchema; +import com.linkedin.data.schema.DataSchemaTraverse; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.PrimitiveDataSchema; +import com.linkedin.data.schema.annotation.SchemaVisitor; +import com.linkedin.data.schema.annotation.SchemaVisitorTraversalResult; +import com.linkedin.data.schema.annotation.TraverserContext; +import com.linkedin.metadata.models.annotation.SearchableRefAnnotation; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; + +/** + * Implementation of {@link SchemaVisitor} responsible for extracting {@link SearchableRefFieldSpec} + * from an aspect schema. + */ +@Slf4j +public class SearchableRefFieldSpecExtractor implements SchemaVisitor { + + private final List _specs = new ArrayList<>(); + private final Map _searchRefFieldNamesToPatch = new HashMap<>(); + + public List getSpecs() { + return _specs; + } + + @Override + public void callbackOnContext(TraverserContext context, DataSchemaTraverse.Order order) { + if (context.getEnclosingField() == null) { + return; + } + + if (DataSchemaTraverse.Order.PRE_ORDER.equals(order)) { + + final DataSchema currentSchema = context.getCurrentSchema().getDereferencedDataSchema(); + + final Object annotationObj = getAnnotationObj(context); + + if (annotationObj != null) { + if (currentSchema.getDereferencedDataSchema().isComplex()) { + final ComplexDataSchema complexSchema = (ComplexDataSchema) currentSchema; + if (isValidComplexType(complexSchema)) { + extractSearchableRefAnnotation(annotationObj, currentSchema, context); + } + } else if (isValidPrimitiveType((PrimitiveDataSchema) currentSchema)) { + extractSearchableRefAnnotation(annotationObj, currentSchema, context); + } else { + throw new ModelValidationException( + String.format( + "Invalid @SearchableRef Annotation at %s", + context.getSchemaPathSpec().toString())); + } + } + } + } + + private Object getAnnotationObj(TraverserContext context) { + final DataSchema currentSchema = context.getCurrentSchema().getDereferencedDataSchema(); + + // First, check properties for primary annotation definition. + final Map properties = context.getEnclosingField().getProperties(); + final Object primaryAnnotationObj = properties.get(SearchableRefAnnotation.ANNOTATION_NAME); + if (primaryAnnotationObj != null) { + validatePropertiesAnnotation( + currentSchema, primaryAnnotationObj, context.getTraversePath().toString()); + + if (currentSchema.getDereferencedType() == DataSchema.Type.MAP + && primaryAnnotationObj instanceof Map + && !((Map) primaryAnnotationObj).isEmpty()) { + return ((Map) primaryAnnotationObj).entrySet().stream().findFirst().get().getValue(); + } + } + + // Next, check resolved properties for annotations on primitives. + final Map resolvedProperties = + FieldSpecUtils.getResolvedProperties(currentSchema, properties); + final Object resolvedAnnotationObj = + resolvedProperties.get(SearchableRefAnnotation.ANNOTATION_NAME); + return resolvedAnnotationObj; + } + + private void extractSearchableRefAnnotation( + final Object annotationObj, final DataSchema currentSchema, final TraverserContext context) { + final PathSpec path = new PathSpec(context.getSchemaPathSpec()); + final Optional fullPath = FieldSpecUtils.getPathSpecWithAspectName(context); + SearchableRefAnnotation annotation = + SearchableRefAnnotation.fromPegasusAnnotationObject( + annotationObj, + FieldSpecUtils.getSchemaFieldName(path), + currentSchema.getDereferencedType(), + path.toString()); + String schemaPathSpec = context.getSchemaPathSpec().toString(); + if (_searchRefFieldNamesToPatch.containsKey(annotation.getFieldName()) + && !_searchRefFieldNamesToPatch.get(annotation.getFieldName()).equals(schemaPathSpec)) { + // Try to use path + String pathName = path.toString().replace('/', '_').replace("*", ""); + if (pathName.startsWith("_")) { + pathName = pathName.replaceFirst("_", ""); + } + + if (_searchRefFieldNamesToPatch.containsKey(pathName) + && !_searchRefFieldNamesToPatch.get(pathName).equals(schemaPathSpec)) { + throw new ModelValidationException( + String.format( + "Entity has multiple searchableRef fields with the same field name %s, path: %s", + annotation.getFieldName(), fullPath.orElse(path))); + } else { + annotation = + new SearchableRefAnnotation( + pathName, + annotation.getFieldType(), + annotation.getBoostScore(), + annotation.getDepth(), + annotation.getRefType(), + annotation.getFieldNameAliases()); + } + } + log.debug("SearchableRef annotation for field: {} : {}", schemaPathSpec, annotation); + final SearchableRefFieldSpec fieldSpec = + new SearchableRefFieldSpec(path, annotation, currentSchema); + _specs.add(fieldSpec); + _searchRefFieldNamesToPatch.put( + annotation.getFieldName(), context.getSchemaPathSpec().toString()); + } + + @Override + public VisitorContext getInitialVisitorContext() { + return null; + } + + @Override + public SchemaVisitorTraversalResult getSchemaVisitorTraversalResult() { + return new SchemaVisitorTraversalResult(); + } + + private void validatePropertiesAnnotation( + DataSchema currentSchema, Object annotationObj, String pathStr) { + + // If primitive, assume the annotation is well formed until resolvedProperties reflects it. + if (currentSchema.isPrimitive() + || currentSchema.getDereferencedType().equals(DataSchema.Type.ENUM) + || currentSchema.getDereferencedType().equals(DataSchema.Type.MAP)) { + return; + } + + // Required override case. If the annotation keys are not overrides, they are incorrect. + if (!Map.class.isAssignableFrom(annotationObj.getClass())) { + throw new ModelValidationException( + String.format( + "Failed to validate @%s annotation declared inside %s: Invalid value type provided (Expected Map)", + SearchableRefAnnotation.ANNOTATION_NAME, pathStr)); + } + + Map annotationMap = (Map) annotationObj; + + if (annotationMap.size() == 0) { + throw new ModelValidationException( + String.format( + "Invalid @SearchableRef Annotation at %s. Annotation placed on invalid field of type %s. Must be placed on primitive field.", + pathStr, currentSchema.getType())); + } + + for (String key : annotationMap.keySet()) { + if (!key.startsWith(Character.toString(PathSpec.SEPARATOR))) { + throw new ModelValidationException( + String.format( + "Invalid @SearchableRef Annotation at %s. Annotation placed on invalid field of type %s. Must be placed on primitive field.", + pathStr, currentSchema.getType())); + } + } + } + + private Boolean isValidComplexType(final ComplexDataSchema schema) { + return DataSchema.Type.ENUM.equals(schema.getDereferencedDataSchema().getDereferencedType()) + || DataSchema.Type.MAP.equals(schema.getDereferencedDataSchema().getDereferencedType()); + } + + private Boolean isValidPrimitiveType(final PrimitiveDataSchema schema) { + return true; + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableRefAnnotation.java b/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableRefAnnotation.java new file mode 100644 index 00000000000000..db6cf46dfc96f7 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableRefAnnotation.java @@ -0,0 +1,121 @@ +package com.linkedin.metadata.models.annotation; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.data.schema.DataSchema; +import com.linkedin.metadata.models.ModelValidationException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import javax.annotation.Nonnull; +import lombok.Value; +import org.apache.commons.lang3.EnumUtils; + +/** Simple object representation of the @SearchableRefAnnotation annotation metadata. */ +@Value +public class SearchableRefAnnotation { + + public static final String FIELD_NAME_ALIASES = "fieldNameAliases"; + public static final String ANNOTATION_NAME = "SearchableRef"; + private static final Set DEFAULT_QUERY_FIELD_TYPES = + ImmutableSet.of( + SearchableAnnotation.FieldType.TEXT, + SearchableAnnotation.FieldType.OBJECT, + SearchableAnnotation.FieldType.TEXT_PARTIAL, + SearchableAnnotation.FieldType.WORD_GRAM, + SearchableAnnotation.FieldType.URN, + SearchableAnnotation.FieldType.URN_PARTIAL); + + // Name of the field in the search index. Defaults to the field name in the schema + String fieldName; + // Type of the field. Defines how the field is indexed and matched + SearchableAnnotation.FieldType fieldType; + // Boost multiplier to the match score. Matches on fields with higher boost score ranks higher + double boostScore; + // defines what depth should be explored of reference object + int depth; + // defines entity type of URN + String refType; + // (Optional) Aliases for this given field that can be used for sorting etc. + List fieldNameAliases; + + @Nonnull + public static SearchableRefAnnotation fromPegasusAnnotationObject( + @Nonnull final Object annotationObj, + @Nonnull final String schemaFieldName, + @Nonnull final DataSchema.Type schemaDataType, + @Nonnull final String context) { + if (!Map.class.isAssignableFrom(annotationObj.getClass())) { + throw new ModelValidationException( + String.format( + "Failed to validate @%s annotation declared at %s: Invalid value type provided (Expected Map)", + ANNOTATION_NAME, context)); + } + + Map map = (Map) annotationObj; + final Optional fieldName = AnnotationUtils.getField(map, "fieldName", String.class); + final Optional fieldType = AnnotationUtils.getField(map, "fieldType", String.class); + if (fieldType.isPresent() + && !EnumUtils.isValidEnum(SearchableAnnotation.FieldType.class, fieldType.get())) { + throw new ModelValidationException( + String.format( + "Failed to validate @%s annotation declared at %s: Invalid field 'fieldType'. Invalid fieldType provided. Valid types are %s", + ANNOTATION_NAME, context, Arrays.toString(SearchableAnnotation.FieldType.values()))); + } + final Optional refType = AnnotationUtils.getField(map, "refType", String.class); + if (!refType.isPresent()) { + throw new ModelValidationException( + String.format( + "Failed to validate @%s annotation declared at %s: " + + "Mandatory input field refType defining the Entity Type is not provided", + ANNOTATION_NAME, context)); + } + final Optional depth = AnnotationUtils.getField(map, "depth", Integer.class); + final Optional boostScore = AnnotationUtils.getField(map, "boostScore", Double.class); + final List fieldNameAliases = getFieldNameAliases(map); + final SearchableAnnotation.FieldType resolvedFieldType = + getFieldType(fieldType, schemaDataType); + return new SearchableRefAnnotation( + fieldName.orElse(schemaFieldName), + resolvedFieldType, + boostScore.orElse(1.0), + depth.orElse(2), + refType.get(), + fieldNameAliases); + } + + private static SearchableAnnotation.FieldType getFieldType( + Optional maybeFieldType, DataSchema.Type schemaDataType) { + if (!maybeFieldType.isPresent()) { + return getDefaultFieldType(schemaDataType); + } + return SearchableAnnotation.FieldType.valueOf(maybeFieldType.get()); + } + + private static SearchableAnnotation.FieldType getDefaultFieldType( + DataSchema.Type schemaDataType) { + switch (schemaDataType) { + case INT: + case FLOAT: + return SearchableAnnotation.FieldType.COUNT; + case MAP: + return SearchableAnnotation.FieldType.KEYWORD; + default: + return SearchableAnnotation.FieldType.TEXT; + } + } + + private static List getFieldNameAliases(Map map) { + final List aliases = new ArrayList<>(); + final Optional fieldNameAliases = + AnnotationUtils.getField(map, FIELD_NAME_ALIASES, List.class); + if (fieldNameAliases.isPresent()) { + for (Object alias : fieldNameAliases.get()) { + aliases.add((String) alias); + } + } + return aliases; + } +} diff --git a/entity-registry/src/test/java/com/linkedin/metadata/models/OpenApiSpecBuilderTest.java b/entity-registry/src/test/java/com/linkedin/metadata/models/OpenApiSpecBuilderTest.java index c482b75956c191..bc39d0b4bb1685 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/models/OpenApiSpecBuilderTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/models/OpenApiSpecBuilderTest.java @@ -108,9 +108,9 @@ public void testOpenApiSpecBuilder() throws Exception { Path.of(getClass().getResource("/").getPath(), "open-api.yaml"), openapiYaml.getBytes(StandardCharsets.UTF_8)); - assertEquals(openAPI.getComponents().getSchemas().size(), 914); - assertEquals(openAPI.getComponents().getParameters().size(), 56); - assertEquals(openAPI.getPaths().size(), 102); + assertEquals(openAPI.getComponents().getSchemas().size(), 930); + assertEquals(openAPI.getComponents().getParameters().size(), 57); + assertEquals(openAPI.getPaths().size(), 104); } private OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) { diff --git a/entity-registry/src/test/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoaderTest.java b/entity-registry/src/test/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoaderTest.java index 030cea3c66ef5f..13582696bde03b 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoaderTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoaderTest.java @@ -99,6 +99,7 @@ private EntityRegistry getBaseEntityRegistry() { Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), (RecordDataSchema) DataSchemaFactory.getInstance().getAspectSchema("datasetKey").get(), DataSchemaFactory.getInstance().getAspectClass("datasetKey").get()); diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index 62490c3c567a97..34fe5493a24be3 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -2,6 +2,8 @@ import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import java.util.Arrays; +import java.util.List; /** Static class containing commonly-used constants across DataHub services. */ public class Constants { @@ -88,6 +90,7 @@ public class Constants { public static final String ENTITY_TYPE_ENTITY_NAME = "entityType"; public static final String FORM_ENTITY_NAME = "form"; public static final String RESTRICTED_ENTITY_NAME = "restricted"; + public static final String BUSINESS_ATTRIBUTE_ENTITY_NAME = "businessAttribute"; /** Aspects */ // Common @@ -379,6 +382,14 @@ public class Constants { public static final String DATA_PROCESS_INSTANCE_RELATIONSHIPS_ASPECT_NAME = "dataProcessInstanceRelationships"; + // Business Attribute + public static final String BUSINESS_ATTRIBUTE_KEY_ASPECT_NAME = "businessAttributeKey"; + public static final String BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME = "businessAttributeInfo"; + public static final String BUSINESS_ATTRIBUTE_ASSOCIATION = "businessAttributeAssociation"; + public static final String BUSINESS_ATTRIBUTE_ASPECT = "businessAttributes"; + public static final List SKIP_REFERENCE_ASPECT = + Arrays.asList("ownership", "status", "institutionalMemory"); + // Posts public static final String POST_INFO_ASPECT_NAME = "postInfo"; public static final String LAST_MODIFIED_FIELD_NAME = "lastModified"; diff --git a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/BusinessAttributeUrn.java b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/BusinessAttributeUrn.java new file mode 100644 index 00000000000000..31893e95e1a979 --- /dev/null +++ b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/BusinessAttributeUrn.java @@ -0,0 +1,70 @@ +package com.linkedin.common.urn; + +import com.linkedin.data.template.Custom; +import com.linkedin.data.template.DirectCoercer; +import com.linkedin.data.template.TemplateOutputCastException; +import java.net.URISyntaxException; + +public final class BusinessAttributeUrn extends Urn { + + public static final String ENTITY_TYPE = "businessAttribute"; + + private final String _name; + + public BusinessAttributeUrn(String name) { + super(ENTITY_TYPE, TupleKey.create(name)); + this._name = name; + } + + public String getName() { + return _name; + } + + public static BusinessAttributeUrn createFromString(String rawUrn) throws URISyntaxException { + return createFromUrn(Urn.createFromString(rawUrn)); + } + + public static BusinessAttributeUrn createFromUrn(Urn urn) throws URISyntaxException { + if (!"li".equals(urn.getNamespace())) { + throw new URISyntaxException(urn.toString(), "Urn namespace type should be 'li'."); + } else if (!ENTITY_TYPE.equals(urn.getEntityType())) { + throw new URISyntaxException( + urn.toString(), "Urn entity type should be '" + urn.getEntityType() + "'."); + } else { + TupleKey key = urn.getEntityKey(); + if (key.size() != 1) { + throw new URISyntaxException( + urn.toString(), "Invalid number of keys: found " + key.size() + " expected 1."); + } else { + try { + return new BusinessAttributeUrn((String) key.getAs(0, String.class)); + } catch (Exception e) { + throw new URISyntaxException(urn.toString(), "Invalid URN Parameter: '" + e.getMessage()); + } + } + } + } + + public static BusinessAttributeUrn deserialize(String rawUrn) throws URISyntaxException { + return createFromString(rawUrn); + } + + static { + Custom.registerCoercer( + new DirectCoercer() { + public Object coerceInput(BusinessAttributeUrn object) throws ClassCastException { + return object.toString(); + } + + public BusinessAttributeUrn coerceOutput(Object object) + throws TemplateOutputCastException { + try { + return BusinessAttributeUrn.createFromString((String) object); + } catch (URISyntaxException e) { + throw new TemplateOutputCastException("Invalid URN syntax: " + e.getMessage(), e); + } + } + }, + BusinessAttributeUrn.class); + } +} diff --git a/li-utils/src/main/pegasus/com/linkedin/common/BusinessAttributeUrn.pdl b/li-utils/src/main/pegasus/com/linkedin/common/BusinessAttributeUrn.pdl new file mode 100644 index 00000000000000..105fb1fefec21c --- /dev/null +++ b/li-utils/src/main/pegasus/com/linkedin/common/BusinessAttributeUrn.pdl @@ -0,0 +1,4 @@ +namespace com.linkedin.common + +@java.class = "com.linkedin.common.urn.BusinessAttributeUrn" +typeref BusinessAttributeUrn = string \ No newline at end of file diff --git a/metadata-ingestion/docs/sources/sigma/sigma_pre.md b/metadata-ingestion/docs/sources/sigma/sigma_pre.md new file mode 100644 index 00000000000000..382a2fe67b944d --- /dev/null +++ b/metadata-ingestion/docs/sources/sigma/sigma_pre.md @@ -0,0 +1,74 @@ +## Integration Details + +This source extracts the following: + +- Workspaces and workbooks within that workspaces as Container. +- Sigma Datasets as Datahub Datasets. +- Pages as Datahub dashboards and elements present inside pages as charts. + +## Configuration Notes + +1. Refer [doc](https://help.sigmacomputing.com/docs/generate-api-client-credentials) to generate an API client credentials. +2. Provide the generated Client ID and Secret in Recipe. + +## Concept mapping + +| Sigma | Datahub | Notes | +|------------------------|---------------------------------------------------------------|----------------------------------| +| `Workspace` | [Container](../../metamodel/entities/container.md) | SubType `"Sigma Workspace"` | +| `Workbook` | [Container](../../metamodel/entities/container.md) | SubType `"Sigma Workbook"` | +| `Page` | [Dashboard](../../metamodel/entities/dashboard.md) | | +| `Element` | [Chart](../../metamodel/entities/chart.md) | | +| `Dataset` | [Dataset](../../metamodel/entities/dataset.md) | SubType `"Sigma Dataset"` | +| `User` | [User (a.k.a CorpUser)](../../metamodel/entities/corpuser.md) | Optionally Extracted | + +## Advanced Configurations + +### Chart source platform mapping +If you want to provide platform details(platform name, platform instance and env) for chart's all external upstream data sources, then you can use `chart_sources_platform_mapping` as below: + +#### Example - For just one specific chart's external upstream data sources +```yml + chart_sources_platform_mapping: + 'workspace_name/workbook_name/chart_name_1': + data_source_platform: snowflake + platform_instance: new_instance + env: PROD + + 'workspace_name/folder_name/workbook_name/chart_name_2': + data_source_platform: postgres + platform_instance: cloud_instance + env: DEV +``` + +#### Example - For all charts within one specific workbook +```yml + chart_sources_platform_mapping: + 'workspace_name/workbook_name_1': + data_source_platform: snowflake + platform_instance: new_instance + env: PROD + + 'workspace_name/folder_name/workbook_name_2': + data_source_platform: snowflake + platform_instance: new_instance + env: PROD +``` + +#### Example - For all workbooks charts within one specific workspace +```yml + chart_sources_platform_mapping: + 'workspace_name': + data_source_platform: snowflake + platform_instance: new_instance + env: PROD +``` + +#### Example - All workbooks use the same connection +```yml + chart_sources_platform_mapping: + '*': + data_source_platform: snowflake + platform_instance: new_instance + env: PROD +``` diff --git a/metadata-ingestion/docs/sources/sigma/sigma_recipe.yml b/metadata-ingestion/docs/sources/sigma/sigma_recipe.yml new file mode 100644 index 00000000000000..c44a9c70b872ef --- /dev/null +++ b/metadata-ingestion/docs/sources/sigma/sigma_recipe.yml @@ -0,0 +1,25 @@ +source: + type: sigma + config: + # Coordinates + api_url: "https://aws-api.sigmacomputing.com/v2" + # Credentials + client_id: "CLIENTID" + client_secret: "CLIENT_SECRET" + + # Optional - filter for certain workspace names instead of ingesting everything. + # workspace_pattern: + # allow: + # - workspace_name + + ingest_owner: true + + # Optional - mapping of sigma workspace/workbook/chart folder path to all chart's data sources platform details present inside that folder path. + # chart_sources_platform_mapping: + # folder_path: + # data_source_platform: postgres + # platform_instance: cloud_instance + # env: DEV + +sink: + # sink configs diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 674450999ad735..b4e19d267af827 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -417,6 +417,7 @@ "databricks": databricks | sql_common | sqllineage_lib, "fivetran": snowflake_common | bigquery_common, "qlik-sense": sqlglot_lib | {"requests", "websocket-client"}, + "sigma": {"requests"}, } # This is mainly used to exclude plugins from the Docker image. @@ -553,6 +554,7 @@ "fivetran", "kafka-connect", "qlik-sense", + "sigma", ] if plugin for dependency in plugins[plugin] @@ -660,6 +662,7 @@ "sql-queries = datahub.ingestion.source.sql_queries:SqlQueriesSource", "fivetran = datahub.ingestion.source.fivetran.fivetran:FivetranSource", "qlik-sense = datahub.ingestion.source.qlik_sense.qlik_sense:QlikSenseSource", + "sigma = datahub.ingestion.source.sigma.sigma:SigmaSource", ], "datahub.ingestion.transformer.plugins": [ "pattern_cleanup_ownership = datahub.ingestion.transformer.pattern_cleanup_ownership:PatternCleanUpOwnership", diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py index b44ab8cc02420d..29f7f786b0a495 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py @@ -17,6 +17,7 @@ class DatasetSubTypes(str, Enum): POWERBI_DATASET_TABLE = "PowerBI Dataset Table" QLIK_DATASET = "Qlik Dataset" BIGQUERY_TABLE_SNAPSHOT = "Bigquery Table Snapshot" + SIGMA_DATASET = "Sigma Dataset" # TODO: Create separate entity... NOTEBOOK = "Notebook" @@ -45,6 +46,8 @@ class BIContainerSubTypes(str, Enum): POWERBI_DATASET = "PowerBI Dataset" QLIK_SPACE = "Qlik Space" QLIK_APP = "Qlik App" + SIGMA_WORKSPACE = "Sigma Workspace" + SIGMA_WORKBOOK = "Sigma Workbook" class JobContainerSubTypes(str, Enum): diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/config.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/config.py new file mode 100644 index 00000000000000..cacb96b4bcca00 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/config.py @@ -0,0 +1,86 @@ +import logging +from dataclasses import dataclass +from typing import Dict, Optional + +import pydantic + +from datahub.configuration.common import AllowDenyPattern +from datahub.configuration.source_common import ( + EnvConfigMixin, + PlatformInstanceConfigMixin, +) +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalSourceReport, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionConfigBase, +) + +logger = logging.getLogger(__name__) + + +class Constant: + """ + keys used in sigma plugin + """ + + # Rest API response key constants + ENTRIES = "entries" + FIRSTNAME = "firstName" + LASTNAME = "lastName" + EDGES = "edges" + DEPENDENCIES = "dependencies" + SOURCE = "source" + WORKSPACEID = "workspaceId" + PATH = "path" + NAME = "name" + URL = "url" + ELEMENTID = "elementId" + ID = "id" + PARENTID = "parentId" + TYPE = "type" + DATASET = "dataset" + WORKBOOK = "workbook" + BADGE = "badge" + NEXTPAGE = "nextPage" + + # Source Config constants + DEFAULT_API_URL = "https://aws-api.sigmacomputing.com/v2" + + +@dataclass +class SigmaSourceReport(StaleEntityRemovalSourceReport): + number_of_workspaces: int = 0 + + def report_number_of_workspaces(self, number_of_workspaces: int) -> None: + self.number_of_workspaces = number_of_workspaces + + +class PlatformDetail(PlatformInstanceConfigMixin, EnvConfigMixin): + data_source_platform: str = pydantic.Field( + description="A chart's data sources platform name.", + ) + + +class SigmaSourceConfig( + StatefulIngestionConfigBase, PlatformInstanceConfigMixin, EnvConfigMixin +): + api_url: str = pydantic.Field( + default=Constant.DEFAULT_API_URL, description="Sigma API hosted URL." + ) + client_id: str = pydantic.Field(description="Sigma Client ID") + client_secret: str = pydantic.Field(description="Sigma Client Secret") + # Sigma workspace identifier + workspace_pattern: AllowDenyPattern = pydantic.Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns to filter Sigma workspaces in ingestion." + "Mention 'User Folder' if entities of 'My documents' need to ingest.", + ) + ingest_owner: Optional[bool] = pydantic.Field( + default=True, + description="Ingest Owner from source. This will override Owner info entered from UI", + ) + chart_sources_platform_mapping: Dict[str, PlatformDetail] = pydantic.Field( + default={}, + description="A mapping of the sigma workspace/workbook/chart folder path to all chart's data sources platform details present inside that folder path.", + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py new file mode 100644 index 00000000000000..9863adc4a854a8 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py @@ -0,0 +1,73 @@ +from datetime import datetime +from typing import Dict, List, Optional + +from pydantic import BaseModel, root_validator + +from datahub.emitter.mcp_builder import ContainerKey + + +class WorkspaceKey(ContainerKey): + workspaceId: str + + +class WorkbookKey(ContainerKey): + workbookId: str + + +class Workspace(BaseModel): + workspaceId: str + name: str + createdBy: str + createdAt: datetime + updatedAt: datetime + + +class SigmaDataset(BaseModel): + datasetId: str + workspaceId: str + name: str + description: str + createdBy: str + createdAt: datetime + updatedAt: datetime + url: str + path: str + badge: Optional[str] = None + + @root_validator(pre=True) + def update_values(cls, values: Dict) -> Dict: + # As element lineage api provide this id as source dataset id + values["datasetId"] = values["url"].split("/")[-1] + return values + + +class Element(BaseModel): + elementId: str + type: str + name: str + url: str + vizualizationType: Optional[str] = None + query: Optional[str] = None + columns: List[str] = [] + upstream_sources: Dict[str, str] = {} + + +class Page(BaseModel): + pageId: str + name: str + elements: List[Element] = [] + + +class Workbook(BaseModel): + workbookId: str + workspaceId: str + name: str + createdBy: str + updatedBy: str + createdAt: datetime + updatedAt: datetime + url: str + path: str + latestVersion: int + pages: List[Page] = [] + badge: Optional[str] = None diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py new file mode 100644 index 00000000000000..263232896e153c --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py @@ -0,0 +1,592 @@ +import logging +from typing import Dict, Iterable, List, Optional + +import datahub.emitter.mce_builder as builder +from datahub.configuration.common import ConfigurationError +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp_builder import add_entity_to_container, gen_containers +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.decorators import ( + SourceCapability, + SupportStatus, + capability, + config_class, + platform_name, + support_status, +) +from datahub.ingestion.api.source import ( + CapabilityReport, + MetadataWorkUnitProcessor, + SourceReport, + TestableSource, + TestConnectionReport, +) +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.common.subtypes import ( + BIContainerSubTypes, + DatasetSubTypes, +) +from datahub.ingestion.source.sigma.config import ( + PlatformDetail, + SigmaSourceConfig, + SigmaSourceReport, +) +from datahub.ingestion.source.sigma.data_classes import ( + Element, + Page, + SigmaDataset, + Workbook, + WorkbookKey, + Workspace, + WorkspaceKey, +) +from datahub.ingestion.source.sigma.sigma_api import SigmaAPI +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalHandler, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionSourceBase, +) +from datahub.metadata.com.linkedin.pegasus2avro.common import ( + Status, + SubTypes, + TimeStamp, +) +from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( + DatasetLineageType, + DatasetProperties, + Upstream, + UpstreamLineage, +) +from datahub.metadata.schema_classes import ( + BrowsePathEntryClass, + BrowsePathsV2Class, + ChangeAuditStampsClass, + ChartInfoClass, + DashboardInfoClass, + DataPlatformInstanceClass, + GlobalTagsClass, + InputFieldClass, + InputFieldsClass, + OwnerClass, + OwnershipClass, + OwnershipTypeClass, + SchemaFieldClass, + SchemaFieldDataTypeClass, + StringTypeClass, + TagAssociationClass, +) +from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result +from datahub.utilities.urns.dataset_urn import DatasetUrn + +# Logger instance +logger = logging.getLogger(__name__) + + +@platform_name("Sigma") +@config_class(SigmaSourceConfig) +@support_status(SupportStatus.INCUBATING) +@capability(SourceCapability.DESCRIPTIONS, "Enabled by default") +@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability( + SourceCapability.OWNERSHIP, + "Enabled by default, configured using `ingest_owner`", +) +class SigmaSource(StatefulIngestionSourceBase, TestableSource): + """ + This plugin extracts the following: + - Sigma Workspaces and Workbooks as Container. + - Sigma Datasets + - Pages as Dashboard and its Elements as Charts + """ + + config: SigmaSourceConfig + reporter: SigmaSourceReport + platform: str = "sigma" + + def __init__(self, config: SigmaSourceConfig, ctx: PipelineContext): + super(SigmaSource, self).__init__(config, ctx) + self.config = config + self.reporter = SigmaSourceReport() + self.dataset_upstream_urn_mapping: Dict[str, List[str]] = {} + try: + self.sigma_api = SigmaAPI(self.config) + except Exception as e: + raise ConfigurationError(f"Unable to connect sigma API. Exception: {e}") + + @staticmethod + def test_connection(config_dict: dict) -> TestConnectionReport: + test_report = TestConnectionReport() + try: + SigmaAPI(SigmaSourceConfig.parse_obj_allow_extras(config_dict)) + test_report.basic_connectivity = CapabilityReport(capable=True) + except Exception as e: + test_report.basic_connectivity = CapabilityReport( + capable=False, failure_reason=str(e) + ) + return test_report + + @classmethod + def create(cls, config_dict, ctx): + config = SigmaSourceConfig.parse_obj(config_dict) + return cls(config, ctx) + + def _gen_workbook_key(self, workbook_id: str) -> WorkbookKey: + return WorkbookKey( + workbookId=workbook_id, + platform=self.platform, + instance=self.config.platform_instance, + ) + + def _gen_workspace_key(self, workspace_id: str) -> WorkspaceKey: + return WorkspaceKey( + workspaceId=workspace_id, + platform=self.platform, + instance=self.config.platform_instance, + ) + + def _get_allowed_workspaces(self) -> List[Workspace]: + all_workspaces = self.sigma_api.workspaces.values() + allowed_workspaces = [ + workspace + for workspace in all_workspaces + if self.config.workspace_pattern.allowed(workspace.name) + ] + logger.info(f"Number of workspaces = {len(all_workspaces)}") + self.reporter.report_number_of_workspaces(len(all_workspaces)) + logger.info(f"Number of allowed workspaces = {len(allowed_workspaces)}") + return allowed_workspaces + + def _gen_workspace_workunit( + self, workspace: Workspace + ) -> Iterable[MetadataWorkUnit]: + """ + Map Sigma workspace to Datahub container + """ + owner_username = self.sigma_api.get_user_name(workspace.createdBy) + yield from gen_containers( + container_key=self._gen_workspace_key(workspace.workspaceId), + name=workspace.name, + sub_types=[BIContainerSubTypes.SIGMA_WORKSPACE], + owner_urn=builder.make_user_urn(owner_username) + if self.config.ingest_owner and owner_username + else None, + created=int(workspace.createdAt.timestamp() * 1000), + last_modified=int(workspace.updatedAt.timestamp() * 1000), + ) + + def _get_sigma_dataset_identifier(self, dataset: SigmaDataset) -> str: + return dataset.datasetId + + def _gen_sigma_dataset_urn(self, dataset_identifier: str) -> str: + return builder.make_dataset_urn_with_platform_instance( + name=dataset_identifier, + env=self.config.env, + platform=self.platform, + platform_instance=self.config.platform_instance, + ) + + def _gen_entity_status_aspect(self, entity_urn: str) -> MetadataWorkUnit: + return MetadataChangeProposalWrapper( + entityUrn=entity_urn, aspect=Status(removed=False) + ).as_workunit() + + def _gen_dataset_properties( + self, dataset_urn: str, dataset: SigmaDataset + ) -> MetadataWorkUnit: + dataset_properties = DatasetProperties( + name=dataset.name, + description=dataset.description, + qualifiedName=dataset.name, + externalUrl=dataset.url, + created=TimeStamp(time=int(dataset.createdAt.timestamp() * 1000)), + lastModified=TimeStamp(time=int(dataset.updatedAt.timestamp() * 1000)), + tags=[dataset.badge] if dataset.badge else None, + ) + dataset_properties.customProperties.update({"path": dataset.path}) + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=dataset_properties + ).as_workunit() + + def _gen_dataplatform_instance_aspect( + self, entity_urn: str + ) -> Optional[MetadataWorkUnit]: + if self.config.platform_instance: + aspect = DataPlatformInstanceClass( + platform=builder.make_data_platform_urn(self.platform), + instance=builder.make_dataplatform_instance_urn( + self.platform, self.config.platform_instance + ), + ) + return MetadataChangeProposalWrapper( + entityUrn=entity_urn, aspect=aspect + ).as_workunit() + else: + return None + + def _gen_entity_owner_aspect( + self, entity_urn: str, user_name: str + ) -> MetadataWorkUnit: + aspect = OwnershipClass( + owners=[ + OwnerClass( + owner=builder.make_user_urn(user_name), + type=OwnershipTypeClass.DATAOWNER, + ) + ] + ) + return MetadataChangeProposalWrapper( + entityUrn=entity_urn, + aspect=aspect, + ).as_workunit() + + def _gen_entity_browsepath_aspect( + self, + entity_urn: str, + parent_entity_urn: str, + paths: List[str], + ) -> MetadataWorkUnit: + entries = [ + BrowsePathEntryClass(id=parent_entity_urn, urn=parent_entity_urn) + ] + [BrowsePathEntryClass(id=path) for path in paths] + if self.config.platform_instance: + urn = builder.make_dataplatform_instance_urn( + self.platform, self.config.platform_instance + ) + entries = [BrowsePathEntryClass(id=urn, urn=urn)] + entries + return MetadataChangeProposalWrapper( + entityUrn=entity_urn, + aspect=BrowsePathsV2Class(entries), + ).as_workunit() + + def _gen_dataset_workunit( + self, dataset: SigmaDataset + ) -> Iterable[MetadataWorkUnit]: + dataset_identifier = self._get_sigma_dataset_identifier(dataset) + dataset_urn = self._gen_sigma_dataset_urn(dataset_identifier) + + yield self._gen_entity_status_aspect(dataset_urn) + + yield self._gen_dataset_properties(dataset_urn, dataset) + + yield from add_entity_to_container( + container_key=self._gen_workspace_key(dataset.workspaceId), + entity_type="dataset", + entity_urn=dataset_urn, + ) + + dpi_aspect = self._gen_dataplatform_instance_aspect(dataset_urn) + if dpi_aspect: + yield dpi_aspect + + owner_username = self.sigma_api.get_user_name(dataset.createdBy) + if self.config.ingest_owner and owner_username: + yield self._gen_entity_owner_aspect(dataset_urn, owner_username) + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=SubTypes(typeNames=[DatasetSubTypes.SIGMA_DATASET]), + ).as_workunit() + + paths = dataset.path.split("/")[1:] + if len(paths) > 0: + yield self._gen_entity_browsepath_aspect( + entity_urn=dataset_urn, + parent_entity_urn=builder.make_container_urn( + self._gen_workspace_key(dataset.workspaceId) + ), + paths=paths, + ) + + if dataset.badge: + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=GlobalTagsClass( + tags=[TagAssociationClass(builder.make_tag_urn(dataset.badge))] + ), + ).as_workunit() + + def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: + return [ + *super().get_workunit_processors(), + StaleEntityRemovalHandler.create( + self, self.config, self.ctx + ).workunit_processor, + ] + + def _gen_dashboard_urn(self, dashboard_identifier: str) -> str: + return builder.make_dashboard_urn( + platform=self.platform, + platform_instance=self.config.platform_instance, + name=dashboard_identifier, + ) + + def _gen_dashboard_info_workunit(self, page: Page) -> MetadataWorkUnit: + dashboard_urn = self._gen_dashboard_urn(page.pageId) + dashboard_info_cls = DashboardInfoClass( + title=page.name, + description="", + charts=[ + builder.make_chart_urn( + platform=self.platform, + platform_instance=self.config.platform_instance, + name=element.elementId, + ) + for element in page.elements + ], + lastModified=ChangeAuditStampsClass(), + customProperties={"ElementsCount": str(len(page.elements))}, + ) + return MetadataChangeProposalWrapper( + entityUrn=dashboard_urn, aspect=dashboard_info_cls + ).as_workunit() + + def _get_element_data_source_platform_details( + self, full_path: str + ) -> Optional[PlatformDetail]: + data_source_platform_details: Optional[PlatformDetail] = None + while full_path != "": + if full_path in self.config.chart_sources_platform_mapping: + data_source_platform_details = ( + self.config.chart_sources_platform_mapping[full_path] + ) + break + else: + full_path = "/".join(full_path.split("/")[:-1]) + if ( + not data_source_platform_details + and "*" in self.config.chart_sources_platform_mapping + ): + data_source_platform_details = self.config.chart_sources_platform_mapping[ + "*" + ] + + return data_source_platform_details + + def _get_element_input_details( + self, element: Element, workbook: Workbook + ) -> Dict[str, List[str]]: + """ + Returns dict with keys as the all element input dataset urn and values as their all upstream dataset urns + """ + inputs: Dict[str, List[str]] = {} + sql_parser_in_tables: List[str] = [] + + data_source_platform_details = self._get_element_data_source_platform_details( + f"{workbook.path}/{workbook.name}/{element.name}" + ) + + if element.query and data_source_platform_details: + try: + sql_parser_in_tables = create_lineage_sql_parsed_result( + query=element.query.strip(), + default_db=None, + platform=data_source_platform_details.data_source_platform, + env=data_source_platform_details.env, + platform_instance=data_source_platform_details.platform_instance, + ).in_tables + except Exception: + logging.debug(f"Unable to parse query of element {element.name}") + + # Add sigma dataset as input of element if present + # and its matched sql parser in_table as its upsteam dataset + for source_id, source_name in element.upstream_sources.items(): + source_id = source_id.split("-")[-1] + for in_table_urn in list(sql_parser_in_tables): + if ( + DatasetUrn.from_string(in_table_urn).name.split(".")[-1] + in source_name.lower() + ): + dataset_urn = self._gen_sigma_dataset_urn(source_id) + if dataset_urn not in inputs: + inputs[dataset_urn] = [in_table_urn] + else: + inputs[dataset_urn].append(in_table_urn) + sql_parser_in_tables.remove(in_table_urn) + + # Add remaining sql parser in_tables as direct input of element + for in_table_urn in sql_parser_in_tables: + inputs[in_table_urn] = [] + + return inputs + + def _gen_elements_workunit( + self, + elements: List[Element], + workbook: Workbook, + all_input_fields: List[InputFieldClass], + ) -> Iterable[MetadataWorkUnit]: + """ + Map Sigma page element to Datahub Chart + """ + + for element in elements: + chart_urn = builder.make_chart_urn( + platform=self.platform, + platform_instance=self.config.platform_instance, + name=element.elementId, + ) + + custom_properties = { + "VizualizationType": str(element.vizualizationType), + "type": str(element.type), + } + + yield self._gen_entity_status_aspect(chart_urn) + + inputs: Dict[str, List[str]] = self._get_element_input_details( + element, workbook + ) + + yield MetadataChangeProposalWrapper( + entityUrn=chart_urn, + aspect=ChartInfoClass( + title=element.name, + description="", + lastModified=ChangeAuditStampsClass(), + customProperties=custom_properties, + externalUrl=element.url, + inputs=list(inputs.keys()), + ), + ).as_workunit() + + yield from add_entity_to_container( + container_key=self._gen_workbook_key(workbook.workbookId), + entity_type="chart", + entity_urn=chart_urn, + ) + + # Add sigma dataset's upstream dataset urn mapping + for dataset_urn, upstream_dataset_urns in inputs.items(): + if ( + upstream_dataset_urns + and dataset_urn not in self.dataset_upstream_urn_mapping + ): + self.dataset_upstream_urn_mapping[ + dataset_urn + ] = upstream_dataset_urns + + element_input_fields = [ + InputFieldClass( + schemaFieldUrn=builder.make_schema_field_urn(chart_urn, column), + schemaField=SchemaFieldClass( + fieldPath=column, + type=SchemaFieldDataTypeClass(StringTypeClass()), + nativeDataType="String", # Make type default as Sting + ), + ) + for column in element.columns + ] + + yield MetadataChangeProposalWrapper( + entityUrn=chart_urn, + aspect=InputFieldsClass(fields=element_input_fields), + ).as_workunit() + + all_input_fields.extend(element_input_fields) + + def _gen_pages_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUnit]: + """ + Map Sigma workbook page to Datahub dashboard + """ + for page in workbook.pages: + dashboard_urn = self._gen_dashboard_urn(page.pageId) + + yield self._gen_entity_status_aspect(dashboard_urn) + + yield self._gen_dashboard_info_workunit(page) + + yield from add_entity_to_container( + container_key=self._gen_workbook_key(workbook.workbookId), + entity_type="dashboard", + entity_urn=dashboard_urn, + ) + + dpi_aspect = self._gen_dataplatform_instance_aspect(dashboard_urn) + if dpi_aspect: + yield dpi_aspect + + all_input_fields: List[InputFieldClass] = [] + + yield from self._gen_elements_workunit( + page.elements, workbook, all_input_fields + ) + + yield MetadataChangeProposalWrapper( + entityUrn=self._gen_dashboard_urn(page.pageId), + aspect=InputFieldsClass(fields=all_input_fields), + ).as_workunit() + + def _gen_workbook_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUnit]: + """ + Map Sigma Workbook to Datahub container + """ + owner_username = self.sigma_api.get_user_name(workbook.createdBy) + yield from gen_containers( + container_key=self._gen_workbook_key(workbook.workbookId), + name=workbook.name, + sub_types=[BIContainerSubTypes.SIGMA_WORKBOOK], + parent_container_key=self._gen_workspace_key(workbook.workspaceId), + extra_properties={ + "path": workbook.path, + "latestVersion": str(workbook.latestVersion), + }, + owner_urn=builder.make_user_urn(owner_username) + if self.config.ingest_owner and owner_username + else None, + external_url=workbook.url, + tags=[workbook.badge] if workbook.badge else None, + created=int(workbook.createdAt.timestamp() * 1000), + last_modified=int(workbook.updatedAt.timestamp() * 1000), + ) + + paths = workbook.path.split("/")[1:] + if len(paths) > 0: + yield self._gen_entity_browsepath_aspect( + entity_urn=builder.make_container_urn( + self._gen_workbook_key(workbook.workbookId), + ), + parent_entity_urn=builder.make_container_urn( + self._gen_workspace_key(workbook.workspaceId) + ), + paths=paths, + ) + + yield from self._gen_pages_workunit(workbook) + + def _gen_sigma_dataset_upstream_lineage_workunit( + self, + ) -> Iterable[MetadataWorkUnit]: + for ( + dataset_urn, + upstream_dataset_urns, + ) in self.dataset_upstream_urn_mapping.items(): + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=UpstreamLineage( + upstreams=[ + Upstream( + dataset=upstream_dataset_urn, type=DatasetLineageType.COPY + ) + for upstream_dataset_urn in upstream_dataset_urns + ], + ), + ).as_workunit() + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + """ + Datahub Ingestion framework invoke this method + """ + logger.info("Sigma plugin execution is started") + entities = self.sigma_api.get_sigma_entities() + for entity in entities: + if isinstance(entity, Workbook): + yield from self._gen_workbook_workunit(entity) + elif isinstance(entity, SigmaDataset): + yield from self._gen_dataset_workunit(entity) + for workspace in self._get_allowed_workspaces(): + yield from self._gen_workspace_workunit(workspace) + yield from self._gen_sigma_dataset_upstream_lineage_workunit() + + def get_report(self) -> SourceReport: + return self.reporter diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma_api.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma_api.py new file mode 100644 index 00000000000000..c335bee15931db --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma_api.py @@ -0,0 +1,259 @@ +import logging +import sys +from typing import Any, Dict, List, Optional, Union + +import requests + +from datahub.ingestion.source.sigma.config import Constant, SigmaSourceConfig +from datahub.ingestion.source.sigma.data_classes import ( + Element, + Page, + SigmaDataset, + Workbook, + Workspace, +) + +# Logger instance +logger = logging.getLogger(__name__) + + +class SigmaAPI: + def __init__(self, config: SigmaSourceConfig) -> None: + self.config = config + self.workspaces: Dict[str, Workspace] = {} + self.users: Dict[str, str] = {} + self.session = requests.Session() + # Test connection by generating access token + logger.info("Trying to connect to {}".format(self.config.api_url)) + self._generate_token() + + def _generate_token(self): + data = { + "grant_type": "client_credentials", + "client_id": self.config.client_id, + "client_secret": self.config.client_secret, + } + response = self.session.post(f"{self.config.api_url}/auth/token", data=data) + response.raise_for_status() + self.session.headers.update( + { + "Authorization": f"Bearer {response.json()['access_token']}", + "Content-Type": "application/json", + } + ) + + def _log_http_error(self, message: str) -> Any: + logger.warning(message) + _, e, _ = sys.exc_info() + if isinstance(e, requests.exceptions.HTTPError): + logger.warning(f"HTTP status-code = {e.response.status_code}") + logger.debug(msg=message, exc_info=e) + return e + + def get_workspace(self, workspace_id: str) -> Optional[Workspace]: + workspace: Optional[Workspace] = None + try: + response = self.session.get( + f"{self.config.api_url}/workspaces/{workspace_id}" + ) + response.raise_for_status() + workspace_dict = response.json() + workspace = Workspace.parse_obj(workspace_dict) + except Exception as e: + self._log_http_error( + message=f"Unable to fetch workspace {workspace_id}. Exception: {e}" + ) + return workspace + + def get_user_name(self, user_id: str) -> Optional[str]: + try: + if user_id in self.users: + # To avoid fetching same user details again + return self.users[user_id] + else: + response = self.session.get(f"{self.config.api_url}/members/{user_id}") + response.raise_for_status() + user_dict = response.json() + user_name = ( + f"{user_dict[Constant.FIRSTNAME]}_{user_dict[Constant.LASTNAME]}" + ) + self.users[user_id] = user_name + return user_name + except Exception as e: + self._log_http_error( + message=f"Unable to fetch user with id {user_id}. Exception: {e}" + ) + return None + + def get_sigma_dataset( + self, dataset_id: str, workspace_id: str, path: str + ) -> Optional[SigmaDataset]: + dataset: Optional[SigmaDataset] = None + try: + response = self.session.get(f"{self.config.api_url}/datasets/{dataset_id}") + response.raise_for_status() + dataset_dict = response.json() + dataset_dict[Constant.WORKSPACEID] = workspace_id + dataset_dict[Constant.PATH] = path + dataset = SigmaDataset.parse_obj(dataset_dict) + except Exception as e: + self._log_http_error( + message=f"Unable to fetch sigma dataset {dataset_id}. Exception: {e}" + ) + return dataset + + def _get_element_upstream_sources( + self, element_id: str, workbook_id: str + ) -> Dict[str, str]: + """ + Returns upstream dataset sources with keys as id and values as name of that dataset + """ + upstream_sources: Dict[str, str] = {} + try: + response = self.session.get( + f"{self.config.api_url}/workbooks/{workbook_id}/lineage/elements/{element_id}" + ) + response.raise_for_status() + response_dict = response.json() + for edge in response_dict[Constant.EDGES]: + source_type = response_dict[Constant.DEPENDENCIES][ + edge[Constant.SOURCE] + ][Constant.TYPE] + if source_type == "dataset": + upstream_sources[edge[Constant.SOURCE]] = response_dict[ + Constant.DEPENDENCIES + ][edge[Constant.SOURCE]][Constant.NAME] + except Exception as e: + self._log_http_error( + message=f"Unable to fetch lineage of element {element_id}. Exception: {e}" + ) + return upstream_sources + + def _get_element_sql_query( + self, element_id: str, workbook_id: str + ) -> Optional[str]: + query: Optional[str] = None + try: + response = self.session.get( + f"{self.config.api_url}/workbooks/{workbook_id}/elements/{element_id}/query" + ) + response.raise_for_status() + response_dict = response.json() + if "sql" in response_dict: + query = response_dict["sql"] + except Exception as e: + self._log_http_error( + message=f"Unable to fetch sql query for a element {element_id}. Exception: {e}" + ) + return query + + def get_page_elements(self, workbook: Workbook, page: Page) -> List[Element]: + elements: List[Element] = [] + try: + response = self.session.get( + f"{self.config.api_url}/workbooks/{workbook.workbookId}/pages/{page.pageId}/elements" + ) + response.raise_for_status() + for i, element_dict in enumerate(response.json()[Constant.ENTRIES]): + if not element_dict.get(Constant.NAME): + element_dict[Constant.NAME] = f"Element {i+1} of Page '{page.name}'" + element_dict[ + Constant.URL + ] = f"{workbook.url}?:nodeId={element_dict[Constant.ELEMENTID]}&:fullScreen=true" + element = Element.parse_obj(element_dict) + element.upstream_sources = self._get_element_upstream_sources( + element.elementId, workbook.workbookId + ) + element.query = self._get_element_sql_query( + element.elementId, workbook.workbookId + ) + elements.append(element) + except Exception as e: + self._log_http_error( + message=f"Unable to fetch elements of page {page.pageId}, workbook {workbook.workbookId}. Exception: {e}" + ) + return elements + + def get_workbook_pages(self, workbook: Workbook) -> List[Page]: + pages: List[Page] = [] + try: + response = self.session.get( + f"{self.config.api_url}/workbooks/{workbook.workbookId}/pages" + ) + response.raise_for_status() + for page_dict in response.json()[Constant.ENTRIES]: + page = Page.parse_obj(page_dict) + page.elements = self.get_page_elements(workbook, page) + pages.append(page) + except Exception as e: + self._log_http_error( + message=f"Unable to fetch pages of workbook {workbook.workbookId}. Exception: {e}" + ) + return pages + + def get_workbook(self, workbook_id: str, workspace_id: str) -> Optional[Workbook]: + workbook: Optional[Workbook] = None + try: + response = self.session.get( + f"{self.config.api_url}/workbooks/{workbook_id}" + ) + response.raise_for_status() + workbook_dict = response.json() + workbook_dict[Constant.WORKSPACEID] = workspace_id + workbook = Workbook.parse_obj(workbook_dict) + workbook.pages = self.get_workbook_pages(workbook) + except Exception as e: + self._log_http_error( + message=f"Unable to fetch workbook {workbook_id}. Exception: {e}" + ) + return workbook + + def get_workspace_id(self, parent_id: str, path: str) -> str: + path_list = path.split("/") + while len(path_list) != 1: # means current parent id is folder's id + response = self.session.get(f"{self.config.api_url}/files/{parent_id}") + parent_id = response.json()[Constant.PARENTID] + path_list.pop() + return parent_id + + def get_sigma_entities(self) -> List[Union[Workbook, SigmaDataset]]: + entities: List[Union[Workbook, SigmaDataset]] = [] + url = f"{self.config.api_url}/files" + while True: + response = self.session.get(url) + response.raise_for_status() + response_dict = response.json() + for entity in response_dict[Constant.ENTRIES]: + workspace_id = self.get_workspace_id( + entity[Constant.PARENTID], entity[Constant.PATH] + ) + if workspace_id not in self.workspaces: + workspace = self.get_workspace(workspace_id) + if workspace: + self.workspaces[workspace.workspaceId] = workspace + + if self.workspaces.get( + workspace_id + ) and self.config.workspace_pattern.allowed( + self.workspaces[workspace_id].name + ): + type = entity[Constant.TYPE] + if type == Constant.DATASET: + dataset = self.get_sigma_dataset( + entity[Constant.ID], + workspace_id, + entity[Constant.PATH], + ) + if dataset: + dataset.badge = entity[Constant.BADGE] + entities.append(dataset) + elif type == Constant.WORKBOOK: + workbook = self.get_workbook(entity[Constant.ID], workspace_id) + if workbook: + workbook.badge = entity[Constant.BADGE] + entities.append(workbook) + if response_dict[Constant.NEXTPAGE]: + url = f"{url}?page={response_dict[Constant.NEXTPAGE]}" + else: + break + return entities diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java index 4322ea90edf1fa..afc831b004ec38 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java @@ -39,6 +39,7 @@ public void reindexAll() { @Override public List buildReindexConfigs() { Map settings = settingsBuilder.getSettings(); + MappingsBuilder.setEntityRegistry(entityRegistry); return entityRegistry.getEntitySpecs().values().stream() .map( entitySpec -> { @@ -57,6 +58,7 @@ public List buildReindexConfigs() { public List buildReindexConfigsWithAllStructProps( Collection properties) { Map settings = settingsBuilder.getSettings(); + MappingsBuilder.setEntityRegistry(entityRegistry); return entityRegistry.getEntitySpecs().values().stream() .map( entitySpec -> { @@ -81,6 +83,7 @@ public List buildReindexConfigsWithAllStructProps( public List buildReindexConfigsWithNewStructProp( StructuredPropertyDefinition property) { Map settings = settingsBuilder.getSettings(); + MappingsBuilder.setEntityRegistry(entityRegistry); return entityRegistry.getEntitySpecs().values().stream() .map( entitySpec -> { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java index 0f1b05b3d0b783..f8d0f165bcddf5 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java @@ -12,7 +12,9 @@ import com.linkedin.metadata.models.LogicalValueType; import com.linkedin.metadata.models.SearchScoreFieldSpec; import com.linkedin.metadata.models.SearchableFieldSpec; +import com.linkedin.metadata.models.SearchableRefFieldSpec; import com.linkedin.metadata.models.annotation.SearchableAnnotation.FieldType; +import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.utils.ESUtils; import com.linkedin.structured.StructuredPropertyDefinition; import java.net.URISyntaxException; @@ -57,6 +59,7 @@ public static Map getPartialNgramConfigWithOverrides( public static final String PROPERTIES = "properties"; public static final String DYNAMIC_TEMPLATES = "dynamic_templates"; + private static EntityRegistry entityRegistry; private MappingsBuilder() {} @@ -119,7 +122,14 @@ public static Map getMappings(@Nonnull final EntitySpec entitySp .forEach( searchScoreFieldSpec -> mappings.putAll(getMappingsForSearchScoreField(searchScoreFieldSpec))); - + entitySpec + .getSearchableRefFieldSpecs() + .forEach( + searchableRefFieldSpec -> + mappings.putAll( + getMappingForSearchableRefField( + searchableRefFieldSpec, + searchableRefFieldSpec.getSearchableRefAnnotation().getDepth()))); // Fixed fields mappings.put("urn", getMappingsForUrn()); mappings.put("runId", getMappingsForRunId()); @@ -307,6 +317,42 @@ private static Map getMappingsForSearchScoreField( ImmutableMap.of(TYPE, ESUtils.DOUBLE_FIELD_TYPE)); } + private static Map getMappingForSearchableRefField( + @Nonnull final SearchableRefFieldSpec searchableRefFieldSpec, @Nonnull final int depth) { + Map mappings = new HashMap<>(); + Map mappingForField = new HashMap<>(); + Map mappingForProperty = new HashMap<>(); + if (depth == 0) { + mappings.put( + searchableRefFieldSpec.getSearchableRefAnnotation().getFieldName(), getMappingsForUrn()); + return mappings; + } + String entityType = searchableRefFieldSpec.getSearchableRefAnnotation().getRefType(); + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityType); + entitySpec + .getSearchableFieldSpecs() + .forEach( + searchableFieldSpec -> + mappingForField.putAll(getMappingsForField(searchableFieldSpec))); + entitySpec + .getSearchableRefFieldSpecs() + .forEach( + entitySearchableRefFieldSpec -> + mappingForField.putAll( + getMappingForSearchableRefField( + entitySearchableRefFieldSpec, + Math.min( + depth - 1, + entitySearchableRefFieldSpec + .getSearchableRefAnnotation() + .getDepth())))); + mappingForField.put("urn", getMappingsForUrn()); + mappingForProperty.put("properties", mappingForField); + mappings.put( + searchableRefFieldSpec.getSearchableRefAnnotation().getFieldName(), mappingForProperty); + return mappings; + } + private static Map getMappingsForFieldNameAliases( @Nonnull final SearchableFieldSpec searchableFieldSpec) { Map mappings = new HashMap<>(); @@ -321,4 +367,8 @@ private static Map getMappingsForFieldNameAliases( }); return mappings; } + + public static void setEntityRegistry(@Nonnull final EntityRegistry entityRegistryInput) { + entityRegistry = entityRegistryInput; + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchFieldConfig.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchFieldConfig.java index d681df00546acb..7415c6e5ce5aa6 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchFieldConfig.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchFieldConfig.java @@ -1,9 +1,17 @@ package com.linkedin.metadata.search.elasticsearch.query.request; +import static com.linkedin.metadata.Constants.SKIP_REFERENCE_ASPECT; import static com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder.*; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.SearchableFieldSpec; +import com.linkedin.metadata.models.SearchableRefFieldSpec; import com.linkedin.metadata.models.annotation.SearchableAnnotation; +import com.linkedin.metadata.models.annotation.SearchableRefAnnotation; +import com.linkedin.metadata.models.registry.EntityRegistry; +import java.util.HashSet; +import java.util.List; import java.util.Set; import javax.annotation.Nonnull; import lombok.Builder; @@ -20,6 +28,7 @@ public class SearchFieldConfig { public static final Set KEYWORD_FIELDS = Set.of("urn", "runId", "_index"); public static final Set PATH_HIERARCHY_FIELDS = Set.of("browsePathV2"); + public static final float URN_BOOST_SCORE = 10.0f; // These should not be used directly since there is a specific // order in which these rules need to be evaluated for exceptions to @@ -80,6 +89,77 @@ public static SearchFieldConfig detectSubFieldType(@Nonnull SearchableFieldSpec return detectSubFieldType(fieldName, boost, fieldType, searchableAnnotation.isQueryByDefault()); } + public static Set detectSubFieldType( + @Nonnull SearchableRefFieldSpec fieldSpec, int depth, EntityRegistry entityRegistry) { + Set fieldConfigs = new HashSet<>(); + final SearchableRefAnnotation searchableRefAnnotation = fieldSpec.getSearchableRefAnnotation(); + String fieldName = searchableRefAnnotation.getFieldName(); + final float boost = (float) searchableRefAnnotation.getBoostScore(); + fieldConfigs.addAll(detectSubFieldType(fieldSpec, depth, entityRegistry, boost, "")); + return fieldConfigs; + } + + public static Set detectSubFieldType( + @Nonnull SearchableRefFieldSpec refFieldSpec, + int depth, + EntityRegistry entityRegistry, + float boostScore, + String prefixFieldName) { + Set fieldConfigs = new HashSet<>(); + final SearchableRefAnnotation searchableRefAnnotation = + refFieldSpec.getSearchableRefAnnotation(); + EntitySpec refEntitySpec = entityRegistry.getEntitySpec(searchableRefAnnotation.getRefType()); + String fieldName = searchableRefAnnotation.getFieldName(); + final SearchableAnnotation.FieldType fieldType = searchableRefAnnotation.getFieldType(); + if (!prefixFieldName.isEmpty()) { + fieldName = prefixFieldName + "." + fieldName; + } + + if (depth == 0) { + // at depth 0 if URN is present then query by default should be true + fieldConfigs.add(detectSubFieldType(fieldName, boostScore, fieldType, true)); + return fieldConfigs; + } + + String urnFieldName = fieldName + ".urn"; + fieldConfigs.add( + detectSubFieldType(urnFieldName, boostScore, SearchableAnnotation.FieldType.URN, true)); + List aspectSpecs = refEntitySpec.getAspectSpecs(); + + for (AspectSpec aspectSpec : aspectSpecs) { + if (!SKIP_REFERENCE_ASPECT.contains(aspectSpec.getName())) { + for (SearchableFieldSpec searchableFieldSpec : aspectSpec.getSearchableFieldSpecs()) { + String refFieldName = searchableFieldSpec.getSearchableAnnotation().getFieldName(); + refFieldName = fieldName + "." + refFieldName; + + final SearchableAnnotation searchableAnnotation = + searchableFieldSpec.getSearchableAnnotation(); + final float refBoost = (float) searchableAnnotation.getBoostScore() * boostScore; + final SearchableAnnotation.FieldType refFieldType = searchableAnnotation.getFieldType(); + fieldConfigs.add( + detectSubFieldTypeForRef( + refFieldName, refBoost, refFieldType, searchableAnnotation.isQueryByDefault())); + } + + for (SearchableRefFieldSpec searchableRefFieldSpec : + aspectSpec.getSearchableRefFieldSpecs()) { + String refFieldName = searchableRefFieldSpec.getSearchableRefAnnotation().getFieldName(); + refFieldName = fieldName + "." + refFieldName; + int newDepth = + Math.min(depth - 1, searchableRefFieldSpec.getSearchableRefAnnotation().getDepth()); + final float refBoost = + (float) searchableRefFieldSpec.getSearchableRefAnnotation().getBoostScore() + * boostScore; + fieldConfigs.addAll( + detectSubFieldType( + searchableRefFieldSpec, newDepth, entityRegistry, refBoost, refFieldName)); + } + } + } + + return fieldConfigs; + } + public static SearchFieldConfig detectSubFieldType( String fieldName, SearchableAnnotation.FieldType fieldType, boolean isQueryByDefault) { return detectSubFieldType(fieldName, DEFAULT_BOOST, fieldType, isQueryByDefault); @@ -101,6 +181,22 @@ public static SearchFieldConfig detectSubFieldType( .build(); } + public static SearchFieldConfig detectSubFieldTypeForRef( + String fieldName, + float boost, + SearchableAnnotation.FieldType fieldType, + boolean isQueryByDefault) { + return SearchFieldConfig.builder() + .fieldName(fieldName) + .boost(boost) + .analyzer(getAnalyzer(fieldName, fieldType)) + .hasKeywordSubfield(hasKeywordSubfieldForRefField(fieldName, fieldType)) + .hasDelimitedSubfield(hasDelimitedSubfieldForRefField(fieldName, fieldType)) + .hasWordGramSubfields(hasWordGramSubfieldsForRefField(fieldType)) + .isQueryByDefault(isQueryByDefault) + .build(); + } + public boolean isKeyword() { return KEYWORD_ANALYZER.equals(analyzer()) || isKeyword(fieldName()); } @@ -128,6 +224,25 @@ private static boolean isKeyword(String fieldName) { return fieldName.endsWith(".keyword") || KEYWORD_FIELDS.contains(fieldName); } + private static boolean hasKeywordSubfieldForRefField( + String fieldName, SearchableAnnotation.FieldType fieldType) { + return !"urn".equals(fieldName) + && !fieldName.endsWith(".urn") + && (TYPES_WITH_DELIMITED_SUBFIELD.contains(fieldType) // if delimited then also has keyword + || TYPES_WITH_KEYWORD_SUBFIELD.contains(fieldType)); + } + + private static boolean hasWordGramSubfieldsForRefField(SearchableAnnotation.FieldType fieldType) { + return TYPES_WITH_WORD_GRAM.contains(fieldType); + } + + private static boolean hasDelimitedSubfieldForRefField( + String fieldName, SearchableAnnotation.FieldType fieldType) { + return (fieldName.endsWith(".urn") + || "urn".equals(fieldName) + || TYPES_WITH_DELIMITED_SUBFIELD.contains(fieldType)); + } + private static String getAnalyzer(String fieldName, SearchableAnnotation.FieldType fieldType) { // order is important if (TYPES_WITH_BROWSE_PATH.contains(fieldType)) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java index 27d733ae6d353f..92f93e732ff3cf 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.search.elasticsearch.query.request; +import static com.linkedin.metadata.Constants.SKIP_REFERENCE_ASPECT; import static com.linkedin.metadata.models.SearchableFieldSpecExtractor.PRIMARY_URN_SEARCH_PROPERTIES; import static com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder.*; import static com.linkedin.metadata.search.elasticsearch.query.request.SearchFieldConfig.*; @@ -16,11 +17,15 @@ import com.linkedin.metadata.config.search.custom.BoolQueryConfiguration; import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration; import com.linkedin.metadata.config.search.custom.QueryConfiguration; +import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.SearchScoreFieldSpec; import com.linkedin.metadata.models.SearchableFieldSpec; +import com.linkedin.metadata.models.SearchableRefFieldSpec; import com.linkedin.metadata.models.annotation.SearchScoreAnnotation; import com.linkedin.metadata.models.annotation.SearchableAnnotation; +import com.linkedin.metadata.models.annotation.SearchableRefAnnotation; +import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.utils.ESUtils; import java.io.IOException; import java.util.ArrayList; @@ -86,6 +91,7 @@ public class SearchQueryBuilder { private final WordGramConfiguration wordGramConfiguration; private final CustomizedQueryHandler customizedQueryHandler; + private EntityRegistry entityRegistry; public SearchQueryBuilder( @Nonnull SearchConfiguration searchConfiguration, @@ -96,6 +102,10 @@ public SearchQueryBuilder( this.customizedQueryHandler = CustomizedQueryHandler.builder(customSearchConfiguration).build(); } + public void setEntityRegistry(EntityRegistry entityRegistry) { + this.entityRegistry = entityRegistry; + } + public QueryBuilder buildQuery( @Nonnull List entitySpecs, @Nonnull String query, boolean fulltext) { QueryConfiguration customQueryConfig = @@ -221,42 +231,72 @@ public Set getFieldsFromEntitySpec(EntitySpec entitySpec) { searchableAnnotation.isQueryByDefault())); if (SearchFieldConfig.detectSubFieldType(fieldSpec).hasWordGramSubfields()) { + addWordGramSearchConfig(fields, searchFieldConfig); + } + } + } + + List searchableRefFieldSpecs = entitySpec.getSearchableRefFieldSpecs(); + for (SearchableRefFieldSpec refFieldSpec : searchableRefFieldSpecs) { + int depth = refFieldSpec.getSearchableRefAnnotation().getDepth(); + Set searchFieldConfig = + SearchFieldConfig.detectSubFieldType(refFieldSpec, depth, entityRegistry); + fields.addAll(searchFieldConfig); + + Map fieldTypeMap = + getAllFieldTypeFromSearchableRef(refFieldSpec, depth, entityRegistry, ""); + for (SearchFieldConfig fieldConfig : searchFieldConfig) { + if (fieldConfig.hasDelimitedSubfield()) { fields.add( - SearchFieldConfig.builder() - .fieldName(searchFieldConfig.fieldName() + ".wordGrams2") - .boost(searchFieldConfig.boost() * wordGramConfiguration.getTwoGramFactor()) - .analyzer(WORD_GRAM_2_ANALYZER) - .hasKeywordSubfield(true) - .hasDelimitedSubfield(true) - .hasWordGramSubfields(true) - .isQueryByDefault(true) - .build()); - fields.add( - SearchFieldConfig.builder() - .fieldName(searchFieldConfig.fieldName() + ".wordGrams3") - .boost(searchFieldConfig.boost() * wordGramConfiguration.getThreeGramFactor()) - .analyzer(WORD_GRAM_3_ANALYZER) - .hasKeywordSubfield(true) - .hasDelimitedSubfield(true) - .hasWordGramSubfields(true) - .isQueryByDefault(true) - .build()); - fields.add( - SearchFieldConfig.builder() - .fieldName(searchFieldConfig.fieldName() + ".wordGrams4") - .boost(searchFieldConfig.boost() * wordGramConfiguration.getFourGramFactor()) - .analyzer(WORD_GRAM_4_ANALYZER) - .hasKeywordSubfield(true) - .hasDelimitedSubfield(true) - .hasWordGramSubfields(true) - .isQueryByDefault(true) - .build()); + SearchFieldConfig.detectSubFieldType( + fieldConfig.fieldName() + ".delimited", + fieldConfig.boost() * partialConfiguration.getFactor(), + fieldTypeMap.get(fieldConfig.fieldName()), + fieldConfig.isQueryByDefault())); + } + + if (fieldConfig.hasWordGramSubfields()) { + addWordGramSearchConfig(fields, fieldConfig); } } } return fields; } + private void addWordGramSearchConfig( + Set fields, SearchFieldConfig searchFieldConfig) { + fields.add( + SearchFieldConfig.builder() + .fieldName(searchFieldConfig.fieldName() + ".wordGrams2") + .boost(searchFieldConfig.boost() * wordGramConfiguration.getTwoGramFactor()) + .analyzer(WORD_GRAM_2_ANALYZER) + .hasKeywordSubfield(true) + .hasDelimitedSubfield(true) + .hasWordGramSubfields(true) + .isQueryByDefault(true) + .build()); + fields.add( + SearchFieldConfig.builder() + .fieldName(searchFieldConfig.fieldName() + ".wordGrams3") + .boost(searchFieldConfig.boost() * wordGramConfiguration.getThreeGramFactor()) + .analyzer(WORD_GRAM_3_ANALYZER) + .hasKeywordSubfield(true) + .hasDelimitedSubfield(true) + .hasWordGramSubfields(true) + .isQueryByDefault(true) + .build()); + fields.add( + SearchFieldConfig.builder() + .fieldName(searchFieldConfig.fieldName() + ".wordGrams4") + .boost(searchFieldConfig.boost() * wordGramConfiguration.getFourGramFactor()) + .analyzer(WORD_GRAM_4_ANALYZER) + .hasKeywordSubfield(true) + .hasDelimitedSubfield(true) + .hasWordGramSubfields(true) + .isQueryByDefault(true) + .build()); + } + private Set getStandardFields(@Nonnull EntitySpec entitySpec) { Set fields = new HashSet<>(); @@ -602,4 +642,55 @@ public float getWordGramFactor(String fieldName) { } throw new IllegalArgumentException(fieldName + " does not end with Grams[2-4]"); } + + // visible for unit test + public Map getAllFieldTypeFromSearchableRef( + SearchableRefFieldSpec refFieldSpec, + int depth, + EntityRegistry entityRegistry, + String prefixField) { + final SearchableRefAnnotation searchableRefAnnotation = + refFieldSpec.getSearchableRefAnnotation(); + // contains fieldName as key and SearchableAnnotation as value + Map fieldNameMap = new HashMap<>(); + EntitySpec refEntitySpec = entityRegistry.getEntitySpec(searchableRefAnnotation.getRefType()); + String fieldName = searchableRefAnnotation.getFieldName(); + final SearchableAnnotation.FieldType fieldType = searchableRefAnnotation.getFieldType(); + if (!prefixField.isEmpty()) { + fieldName = prefixField + "." + fieldName; + } + + if (depth == 0) { + // at depth 0 only URN is present then add and return + fieldNameMap.put(fieldName, fieldType); + return fieldNameMap; + } + String urnFieldName = fieldName + ".urn"; + fieldNameMap.put(urnFieldName, SearchableAnnotation.FieldType.URN); + List aspectSpecs = refEntitySpec.getAspectSpecs(); + for (AspectSpec aspectSpec : aspectSpecs) { + if (!SKIP_REFERENCE_ASPECT.contains(aspectSpec.getName())) { + for (SearchableFieldSpec searchableFieldSpec : aspectSpec.getSearchableFieldSpecs()) { + String refFieldName = searchableFieldSpec.getSearchableAnnotation().getFieldName(); + refFieldName = fieldName + "." + refFieldName; + final SearchableAnnotation searchableAnnotation = + searchableFieldSpec.getSearchableAnnotation(); + final SearchableAnnotation.FieldType refFieldType = searchableAnnotation.getFieldType(); + fieldNameMap.put(refFieldName, refFieldType); + } + + for (SearchableRefFieldSpec searchableRefFieldSpec : + aspectSpec.getSearchableRefFieldSpecs()) { + String refFieldName = searchableRefFieldSpec.getSearchableRefAnnotation().getFieldName(); + refFieldName = fieldName + "." + refFieldName; + int newDepth = + Math.min(depth - 1, searchableRefFieldSpec.getSearchableRefAnnotation().getDepth()); + fieldNameMap.putAll( + getAllFieldTypeFromSearchableRef( + searchableRefFieldSpec, newDepth, entityRegistry, refFieldName)); + } + } + } + return fieldNameMap; + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java index 83e83cd73b5a00..5ad024c211d308 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java @@ -118,6 +118,7 @@ private SearchRequestHandler( return set1; })); this.aspectRetriever = aspectRetriever; + searchQueryBuilder.setEntityRegistry(this.aspectRetriever.getEntityRegistry()); } public static SearchRequestHandler getBuilder( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java index b1d8cc075f387b..d1c9b4cdc266f1 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java @@ -5,25 +5,31 @@ import static com.linkedin.metadata.models.annotation.SearchableAnnotation.OBJECT_FIELD_TYPES; import static com.linkedin.metadata.search.elasticsearch.indexbuilder.MappingsBuilder.SYSTEM_CREATED_FIELD; +import com.datahub.util.RecordUtils; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; +import com.linkedin.data.DataMap; import com.linkedin.data.schema.DataSchema; import com.linkedin.data.template.RecordTemplate; import com.linkedin.entity.Aspect; import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.validation.StructuredPropertiesValidator; +import com.linkedin.metadata.entity.EntityUtils; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.LogicalValueType; import com.linkedin.metadata.models.SearchScoreFieldSpec; import com.linkedin.metadata.models.SearchableFieldSpec; +import com.linkedin.metadata.models.SearchableRefFieldSpec; import com.linkedin.metadata.models.annotation.SearchableAnnotation.FieldType; import com.linkedin.metadata.models.extractor.FieldExtractor; +import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.r2.RemoteInvocationException; import com.linkedin.structured.StructuredProperties; import com.linkedin.structured.StructuredPropertyDefinition; @@ -112,17 +118,24 @@ public Optional transformAspect( throws RemoteInvocationException, URISyntaxException { final Map> extractedSearchableFields = FieldExtractor.extractFields(aspect, aspectSpec.getSearchableFieldSpecs(), maxValueLength); + final Map> extractedSearchRefFields = + FieldExtractor.extractFields( + aspect, aspectSpec.getSearchableRefFieldSpecs(), maxValueLength); final Map> extractedSearchScoreFields = FieldExtractor.extractFields(aspect, aspectSpec.getSearchScoreFieldSpecs(), maxValueLength); Optional result = Optional.empty(); - if (!extractedSearchableFields.isEmpty() || !extractedSearchScoreFields.isEmpty()) { + if (!extractedSearchableFields.isEmpty() + || !extractedSearchScoreFields.isEmpty() + || !extractedSearchRefFields.isEmpty()) { final ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); searchDocument.put("urn", urn.toString()); extractedSearchableFields.forEach( (key, values) -> setSearchableValue(key, values, searchDocument, forDelete)); + extractedSearchRefFields.forEach( + (key, values) -> setSearchableRefValue(key, values, searchDocument, forDelete)); extractedSearchScoreFields.forEach( (key, values) -> setSearchScoreValue(key, values, searchDocument, forDelete)); result = Optional.of(searchDocument); @@ -424,4 +437,132 @@ private void setStructuredPropertiesSearchValue( } }); } + + public void setSearchableRefValue( + final SearchableRefFieldSpec searchableRefFieldSpec, + final List fieldValues, + final ObjectNode searchDocument, + final Boolean forDelete) { + String fieldName = searchableRefFieldSpec.getSearchableRefAnnotation().getFieldName(); + FieldType fieldType = searchableRefFieldSpec.getSearchableRefAnnotation().getFieldType(); + boolean isArray = searchableRefFieldSpec.isArray(); + + if (forDelete) { + searchDocument.set(fieldName, JsonNodeFactory.instance.nullNode()); + return; + } + int depth = searchableRefFieldSpec.getSearchableRefAnnotation().getDepth(); + if (isArray) { + ArrayNode arrayNode = JsonNodeFactory.instance.arrayNode(); + fieldValues + .subList(0, Math.min(fieldValues.size(), maxArrayLength)) + .forEach(value -> getNodeForRef(depth, value, fieldType).ifPresent(arrayNode::add)); + searchDocument.set(fieldName, arrayNode); + } else if (!fieldValues.isEmpty()) { + String finalFieldName = fieldName; + getNodeForRef(depth, fieldValues.get(0), fieldType) + .ifPresent(node -> searchDocument.set(finalFieldName, node)); + } else { + searchDocument.set(fieldName, JsonNodeFactory.instance.nullNode()); + } + } + + private Optional getNodeForRef( + final int depth, final Object fieldValue, final FieldType fieldType) { + EntityRegistry entityRegistry = aspectRetriever.getEntityRegistry(); + if (depth == 0) { + if (fieldValue.toString().isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(JsonNodeFactory.instance.textNode(fieldValue.toString())); + } + } + if (fieldType == FieldType.URN) { + ObjectNode resultNode = JsonNodeFactory.instance.objectNode(); + try { + Urn eAUrn = EntityUtils.getUrnFromString(fieldValue.toString()); + String entityType = eAUrn.getEntityType(); + String entityKeyAspectName = entityRegistry.getEntitySpec(entityType).getKeyAspectName(); + Optional entityKeyAspect = + Optional.ofNullable(aspectRetriever.getLatestAspectObject(eAUrn, entityKeyAspectName)); + if (entityKeyAspect.isEmpty()) { + return Optional.ofNullable(JsonNodeFactory.instance.nullNode()); + } + resultNode.set("urn", JsonNodeFactory.instance.textNode(fieldValue.toString())); + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityType); + for (Map.Entry mapEntry : entitySpec.getAspectSpecMap().entrySet()) { + String aspectName = mapEntry.getKey(); + AspectSpec aspectSpec = mapEntry.getValue(); + String aspectClass = aspectSpec.getDataTemplateClass().getCanonicalName(); + if (!Constants.SKIP_REFERENCE_ASPECT.contains(aspectName)) { + try { + Aspect aspectDetails = aspectRetriever.getLatestAspectObject(eAUrn, aspectName); + DataMap aspectDataMap = aspectDetails.data(); + RecordTemplate aspectRecord = + RecordUtils.toRecordTemplate(aspectClass, aspectDataMap); + // Extract searchable fields and create node using getNodeForSearchable + final Map> extractedSearchableFields = + FieldExtractor.extractFields( + aspectRecord, aspectSpec.getSearchableFieldSpecs(), maxValueLength); + for (Map.Entry> entry : + extractedSearchableFields.entrySet()) { + SearchableFieldSpec spec = entry.getKey(); + List value = entry.getValue(); + if (!value.isEmpty()) { + setSearchableValue(spec, value, resultNode, false); + } + } + + // Extract searchable ref fields and create node using getNodeForRef + final Map> extractedSearchableRefFields = + FieldExtractor.extractFields( + aspectDetails, aspectSpec.getSearchableRefFieldSpecs(), maxValueLength); + for (Map.Entry> entry : + extractedSearchableRefFields.entrySet()) { + SearchableRefFieldSpec spec = entry.getKey(); + List value = entry.getValue(); + String fieldName = spec.getSearchableRefAnnotation().getFieldName(); + boolean isArray = spec.isArray(); + if (!value.isEmpty()) { + int newDepth = Math.min(depth - 1, spec.getSearchableRefAnnotation().getDepth()); + if (isArray) { + ArrayNode arrayNode = JsonNodeFactory.instance.arrayNode(); + value + .subList(0, Math.min(value.size(), maxArrayLength)) + .forEach( + val -> + getNodeForRef( + newDepth, + val, + spec.getSearchableRefAnnotation().getFieldType()) + .ifPresent(arrayNode::add)); + resultNode.set(fieldName, arrayNode); + } else { + Optional node = + getNodeForRef( + newDepth, + value.get(0), + spec.getSearchableRefAnnotation().getFieldType()); + if (node.isPresent()) { + resultNode.set(fieldName, node.get()); + } + } + } + } + } catch (RemoteInvocationException e) { + log.error( + "Error while fetching aspect details of {} for urn {} : {}", + aspectName, + eAUrn, + e.getMessage()); + } + } + } + return Optional.of(resultNode); + } catch (Exception e) { + log.error("Error while processing ref field of urn {} : {}", fieldValue, e.getMessage()); + } + } + return Optional.empty(); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java index 6f905b8d31f3f1..b9b2ef6a78e910 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java @@ -110,6 +110,9 @@ public class ESUtils { "fieldDescriptions", ImmutableList.of("fieldDescriptions", "editedFieldDescriptions")); put("description", ImmutableList.of("description", "editedDescription")); + put( + "businessAttribute", + ImmutableList.of("businessAttributeRef", "businessAttributeRef.urn")); } }; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java new file mode 100644 index 00000000000000..c12a1be0d96ac1 --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java @@ -0,0 +1,127 @@ +package com.linkedin.metadata.service; + +import static com.linkedin.metadata.search.utils.QueryUtils.EMPTY_FILTER; +import static com.linkedin.metadata.search.utils.QueryUtils.newFilter; +import static com.linkedin.metadata.search.utils.QueryUtils.newRelationshipFilter; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.businessattribute.BusinessAttributes; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.graph.GraphService; +import com.linkedin.metadata.graph.RelatedEntitiesResult; +import com.linkedin.metadata.graph.RelatedEntity; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.PlatformEvent; +import com.linkedin.platform.event.v1.EntityChangeEvent; +import java.util.Arrays; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class BusinessAttributeUpdateHookService { + private static final String BUSINESS_ATTRIBUTE_OF = "BusinessAttributeOf"; + + private final GraphService graphService; + private final EntityService entityService; + private final EntityRegistry entityRegistry; + + private final int relatedEntitiesCount; + + public static final String TAG = "TAG"; + public static final String GLOSSARY_TERM = "GLOSSARY_TERM"; + public static final String DOCUMENTATION = "DOCUMENTATION"; + + public BusinessAttributeUpdateHookService( + GraphService graphService, + EntityService entityService, + EntityRegistry entityRegistry, + @NonNull @Value("${businessAttribute.fetchRelatedEntitiesCount}") int relatedEntitiesCount) { + this.graphService = graphService; + this.entityService = entityService; + this.entityRegistry = entityRegistry; + this.relatedEntitiesCount = relatedEntitiesCount; + } + + public void handleChangeEvent(@NonNull final PlatformEvent event) { + final EntityChangeEvent entityChangeEvent = + GenericRecordUtils.deserializePayload( + event.getPayload().getValue(), EntityChangeEvent.class); + + if (!entityChangeEvent.getEntityType().equals(Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME)) { + log.info("Skipping MCL event for entity:" + entityChangeEvent.getEntityType()); + return; + } + + final Set businessAttributeCategories = + ImmutableSet.of(TAG, GLOSSARY_TERM, DOCUMENTATION); + if (!businessAttributeCategories.contains(entityChangeEvent.getCategory())) { + log.info("Skipping MCL event for category: " + entityChangeEvent.getCategory()); + return; + } + + Urn urn = entityChangeEvent.getEntityUrn(); + log.info("Business Attribute update hook invoked for urn :" + urn); + + RelatedEntitiesResult entityAssociatedWithBusinessAttribute = + graphService.findRelatedEntities( + null, + newFilter("urn", urn.toString()), + null, + EMPTY_FILTER, + Arrays.asList(BUSINESS_ATTRIBUTE_OF), + newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.INCOMING), + 0, + relatedEntitiesCount); + + for (RelatedEntity relatedEntity : entityAssociatedWithBusinessAttribute.getEntities()) { + String entityUrnStr = relatedEntity.getUrn(); + try { + Urn entityUrn = new Urn(entityUrnStr); + final AspectSpec aspectSpec = + entityRegistry + .getEntitySpec(Constants.SCHEMA_FIELD_ENTITY_NAME) + .getAspectSpec(Constants.BUSINESS_ATTRIBUTE_ASPECT); + + EnvelopedAspect envelopedAspect = + entityService.getLatestEnvelopedAspect( + Constants.SCHEMA_FIELD_ENTITY_NAME, entityUrn, Constants.BUSINESS_ATTRIBUTE_ASPECT); + BusinessAttributes businessAttributes = + new BusinessAttributes(envelopedAspect.getValue().data()); + + final AuditStamp auditStamp = + new AuditStamp() + .setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)) + .setTime(System.currentTimeMillis()); + + entityService + .alwaysProduceMCLAsync( + entityUrn, + Constants.SCHEMA_FIELD_ENTITY_NAME, + Constants.BUSINESS_ATTRIBUTE_ASPECT, + aspectSpec, + null, + businessAttributes, + null, + null, + auditStamp, + ChangeType.RESTATE) + .getFirst(); + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/entity/BusinessAttributeAssociationChangeEvent.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/entity/BusinessAttributeAssociationChangeEvent.java new file mode 100644 index 00000000000000..6749f44b3ee52e --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/data/entity/BusinessAttributeAssociationChangeEvent.java @@ -0,0 +1,43 @@ +package com.linkedin.metadata.timeline.data.entity; + +import com.google.common.collect.ImmutableMap; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.timeline.data.ChangeCategory; +import com.linkedin.metadata.timeline.data.ChangeEvent; +import com.linkedin.metadata.timeline.data.ChangeOperation; +import com.linkedin.metadata.timeline.data.SemanticChangeType; +import java.util.Map; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Value; +import lombok.experimental.NonFinal; + +@EqualsAndHashCode(callSuper = true) +@Value +@NonFinal +@Getter +public class BusinessAttributeAssociationChangeEvent extends ChangeEvent { + @Builder(builderMethodName = "entityBusinessAttributeAssociationChangeEventBuilder") + public BusinessAttributeAssociationChangeEvent( + String entityUrn, + ChangeCategory category, + ChangeOperation operation, + String modifier, + Map parameters, + AuditStamp auditStamp, + SemanticChangeType semVerChange, + String description, + Urn businessAttributeUrn) { + super( + entityUrn, + category, + operation, + modifier, + ImmutableMap.of("businessAttributeUrn", businessAttributeUrn.toString()), + auditStamp, + semVerChange, + description); + } +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributeAssociationChangeEventGenerator.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributeAssociationChangeEventGenerator.java new file mode 100644 index 00000000000000..f0369bc4ace136 --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributeAssociationChangeEventGenerator.java @@ -0,0 +1,82 @@ +package com.linkedin.metadata.timeline.eventgenerator; + +import com.linkedin.businessattribute.BusinessAttributeAssociation; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.timeline.data.ChangeCategory; +import com.linkedin.metadata.timeline.data.ChangeEvent; +import com.linkedin.metadata.timeline.data.ChangeOperation; +import com.linkedin.metadata.timeline.data.SemanticChangeType; +import com.linkedin.metadata.timeline.data.entity.BusinessAttributeAssociationChangeEvent; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import javax.annotation.Nonnull; + +public class BusinessAttributeAssociationChangeEventGenerator + extends EntityChangeEventGenerator { + + private static final String BUSINESS_ATTRIBUTE_ADDED_FORMAT = + "BusinessAttribute '%s' added to entity '%s'."; + private static final String BUSINESS_ATTRIBUTE_REMOVED_FORMAT = + "BusinessAttribute '%s' removed from entity '%s'."; + + public static List computeDiffs( + BusinessAttributeAssociation baseAssociation, + BusinessAttributeAssociation targetAssociation, + String urn, + AuditStamp auditStamp) { + List changeEvents = new ArrayList<>(); + + if (Objects.nonNull(baseAssociation) && Objects.isNull(targetAssociation)) { + changeEvents.add( + createChangeEvent( + baseAssociation, + urn, + ChangeOperation.REMOVE, + BUSINESS_ATTRIBUTE_REMOVED_FORMAT, + auditStamp)); + + } else if (Objects.isNull(baseAssociation) && Objects.nonNull(targetAssociation)) { + changeEvents.add( + createChangeEvent( + targetAssociation, + urn, + ChangeOperation.ADD, + BUSINESS_ATTRIBUTE_ADDED_FORMAT, + auditStamp)); + } + return changeEvents; + } + + private static ChangeEvent createChangeEvent( + BusinessAttributeAssociation association, + String entityUrn, + ChangeOperation operation, + String format, + AuditStamp auditStamp) { + return BusinessAttributeAssociationChangeEvent + .entityBusinessAttributeAssociationChangeEventBuilder() + .modifier(association.getBusinessAttributeUrn().toString()) + .entityUrn(entityUrn) + .category(ChangeCategory.BUSINESS_ATTRIBUTE) + .operation(operation) + .semVerChange(SemanticChangeType.MINOR) + .description( + String.format(format, association.getBusinessAttributeUrn().getId(), entityUrn)) + .businessAttributeUrn(association.getBusinessAttributeUrn()) + .auditStamp(auditStamp) + .build(); + } + + @Override + public List getChangeEvents( + @Nonnull Urn urn, + @Nonnull String entity, + @Nonnull String aspect, + @Nonnull Aspect from, + @Nonnull Aspect to, + @Nonnull AuditStamp auditStamp) { + return computeDiffs(from.getValue(), to.getValue(), urn.toString(), auditStamp); + } +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributeInfoChangeEventGenerator.java b/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributeInfoChangeEventGenerator.java new file mode 100644 index 00000000000000..d797c2d1668d9d --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/BusinessAttributeInfoChangeEventGenerator.java @@ -0,0 +1,148 @@ +package com.linkedin.metadata.timeline.eventgenerator; + +import com.linkedin.businessattribute.BusinessAttributeInfo; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.GlobalTags; +import com.linkedin.common.GlossaryTerms; +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.timeline.data.ChangeCategory; +import com.linkedin.metadata.timeline.data.ChangeEvent; +import com.linkedin.metadata.timeline.data.ChangeOperation; +import com.linkedin.metadata.timeline.data.SemanticChangeType; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import javax.annotation.Nonnull; + +public class BusinessAttributeInfoChangeEventGenerator + extends EntityChangeEventGenerator { + + public static final String ATTRIBUTE_DOCUMENTATION_ADDED_FORMAT = + "Documentation for the businessAttribute '%s' has been added: '%s'"; + public static final String ATTRIBUTE_DOCUMENTATION_REMOVED_FORMAT = + "Documentation for the businessAttribute '%s' has been removed: '%s'"; + public static final String ATTRIBUTE_DOCUMENTATION_UPDATED_FORMAT = + "Documentation for the businessAttribute '%s' has been updated from '%s' to '%s'."; + + @Override + public List getChangeEvents( + @Nonnull Urn urn, + @Nonnull String entity, + @Nonnull String aspect, + @Nonnull Aspect from, + @Nonnull Aspect to, + @Nonnull AuditStamp auditStamp) { + final List changeEvents = new ArrayList<>(); + changeEvents.addAll( + getDocumentationChangeEvent(from.getValue(), to.getValue(), urn.toString(), auditStamp)); + changeEvents.addAll( + getGlossaryTermChangeEvents(from.getValue(), to.getValue(), urn.toString(), auditStamp)); + changeEvents.addAll( + getTagChangeEvents(from.getValue(), to.getValue(), urn.toString(), auditStamp)); + return changeEvents; + } + + private List getDocumentationChangeEvent( + BusinessAttributeInfo baseBusinessAttributeInfo, + BusinessAttributeInfo targetBusinessAttributeInfo, + String entityUrn, + AuditStamp auditStamp) { + String baseDescription = + (baseBusinessAttributeInfo != null) ? baseBusinessAttributeInfo.getDescription() : null; + String targetDescription = + (targetBusinessAttributeInfo != null) ? targetBusinessAttributeInfo.getDescription() : null; + List changeEvents = new ArrayList<>(); + if (baseDescription == null && targetDescription != null) { + changeEvents.add( + createChangeEvent( + targetBusinessAttributeInfo, + entityUrn, + ChangeOperation.ADD, + ATTRIBUTE_DOCUMENTATION_ADDED_FORMAT, + auditStamp, + targetDescription)); + } + + if (baseDescription != null && targetDescription == null) { + changeEvents.add( + createChangeEvent( + baseBusinessAttributeInfo, + entityUrn, + ChangeOperation.REMOVE, + ATTRIBUTE_DOCUMENTATION_REMOVED_FORMAT, + auditStamp, + baseDescription)); + } + + if (baseDescription != null && !baseDescription.equals(targetDescription)) { + changeEvents.add( + createChangeEvent( + targetBusinessAttributeInfo, + entityUrn, + ChangeOperation.MODIFY, + ATTRIBUTE_DOCUMENTATION_UPDATED_FORMAT, + auditStamp, + baseDescription, + targetDescription)); + } + + return changeEvents; + } + + private List getGlossaryTermChangeEvents( + BusinessAttributeInfo baseBusinessAttributeInfo, + BusinessAttributeInfo targetBusinessAttributeInfo, + String entityUrn, + AuditStamp auditStamp) { + GlossaryTerms baseGlossaryTerms = + (baseBusinessAttributeInfo != null) ? baseBusinessAttributeInfo.getGlossaryTerms() : null; + GlossaryTerms targetGlossaryTerms = + (targetBusinessAttributeInfo != null) + ? targetBusinessAttributeInfo.getGlossaryTerms() + : null; + + List entityGlossaryTermsChangeEvents = + GlossaryTermsChangeEventGenerator.computeDiffs( + baseGlossaryTerms, targetGlossaryTerms, entityUrn.toString(), auditStamp); + + return entityGlossaryTermsChangeEvents; + } + + private List getTagChangeEvents( + BusinessAttributeInfo baseBusinessAttributeInfo, + BusinessAttributeInfo targetBusinessAttributeInfo, + String entityUrn, + AuditStamp auditStamp) { + GlobalTags baseGlobalTags = + (baseBusinessAttributeInfo != null) ? baseBusinessAttributeInfo.getGlobalTags() : null; + GlobalTags targetGlobalTags = + (targetBusinessAttributeInfo != null) ? targetBusinessAttributeInfo.getGlobalTags() : null; + + List entityTagChangeEvents = + GlobalTagsChangeEventGenerator.computeDiffs( + baseGlobalTags, targetGlobalTags, entityUrn.toString(), auditStamp); + + return entityTagChangeEvents; + } + + private ChangeEvent createChangeEvent( + BusinessAttributeInfo businessAttributeInfo, + String entityUrn, + ChangeOperation operation, + String format, + AuditStamp auditStamp, + String... descriptions) { + List args = new ArrayList<>(); + args.add(0, businessAttributeInfo.getFieldPath()); + Arrays.stream(descriptions).forEach(val -> args.add(val)); + return ChangeEvent.builder() + .modifier(businessAttributeInfo.getFieldPath()) + .entityUrn(entityUrn) + .category(ChangeCategory.DOCUMENTATION) + .operation(operation) + .semVerChange(SemanticChangeType.MINOR) + .description(String.format(format, args.toArray())) + .auditStamp(auditStamp) + .build(); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/TestSearchFieldConfig.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/TestSearchFieldConfig.java new file mode 100644 index 00000000000000..062298796dd7c7 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/query/request/TestSearchFieldConfig.java @@ -0,0 +1,64 @@ +package com.linkedin.metadata.search.elasticsearch.query.request; + +import com.linkedin.metadata.models.SearchableRefFieldSpec; +import com.linkedin.metadata.models.registry.ConfigEntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistry; +import java.util.Optional; +import java.util.Set; +import org.junit.jupiter.api.Assertions; +import org.testng.annotations.Test; + +@Test +public class TestSearchFieldConfig { + + void setup() {} + + /** + * + * + *
    + *
  • {@link SearchFieldConfig#detectSubFieldType( SearchableRefFieldSpec, int, EntityRegistry + * ) } + *
+ */ + @Test + public void detectSubFieldType() { + EntityRegistry entityRegistry = getTestEntityRegistry(); + SearchableRefFieldSpec searchableRefFieldSpec = + entityRegistry.getEntitySpec("testRefEntity").getSearchableRefFieldSpecs().get(0); + + Set responseForNonZeroDepth = + SearchFieldConfig.detectSubFieldType(searchableRefFieldSpec, 1, entityRegistry); + Assertions.assertTrue( + responseForNonZeroDepth.stream() + .anyMatch( + searchFieldConfig -> + searchFieldConfig.fieldName().equals("refEntityUrns.displayName"))); + Assertions.assertTrue( + responseForNonZeroDepth.stream() + .anyMatch( + searchFieldConfig -> searchFieldConfig.fieldName().equals("refEntityUrns.urn"))); + Assertions.assertTrue( + responseForNonZeroDepth.stream() + .anyMatch( + searchFieldConfig -> + searchFieldConfig.fieldName().equals("refEntityUrns.editedFieldDescriptions"))); + + Set responseForZeroDepth = + SearchFieldConfig.detectSubFieldType(searchableRefFieldSpec, 0, entityRegistry); + Optional searchFieldConfigToCompare = + responseForZeroDepth.stream() + .filter(searchFieldConfig -> searchFieldConfig.fieldName().equals("refEntityUrns")) + .findFirst(); + + Assertions.assertTrue(searchFieldConfigToCompare.isPresent()); + Assertions.assertEquals("query_urn_component", searchFieldConfigToCompare.get().analyzer()); + } + + private EntityRegistry getTestEntityRegistry() { + return new ConfigEntityRegistry( + TestSearchFieldConfig.class + .getClassLoader() + .getResourceAsStream("test-entity-registry.yaml")); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java index 1b4b7de5bf8179..9185e2e7ee072d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java @@ -3,12 +3,19 @@ import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.*; +import com.datahub.test.TestRefEntity; import com.google.common.collect.ImmutableMap; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; import com.linkedin.metadata.TestEntitySpecBuilder; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.EntitySpecBuilder; +import com.linkedin.metadata.models.registry.ConfigEntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.elasticsearch.indexbuilder.MappingsBuilder; +import com.linkedin.metadata.search.elasticsearch.query.request.TestSearchFieldConfig; import com.linkedin.structured.StructuredPropertyDefinition; +import java.io.Serializable; import java.net.URISyntaxException; import java.util.List; import java.util.Map; @@ -272,4 +279,61 @@ public void testGetMappingsForStructuredProperty() throws URISyntaxException { mappings = structuredPropertyFieldMappingsNumber.get(keyInMap); assertEquals(Map.of("type", "double"), mappings); } + + @Test + public void testRefMappingsBuilder() { + EntityRegistry entityRegistry = getTestEntityRegistry(); + MappingsBuilder.setEntityRegistry(entityRegistry); + EntitySpec entitySpec = new EntitySpecBuilder().buildEntitySpec(new TestRefEntity().schema()); + Map result = MappingsBuilder.getMappings(entitySpec); + assertEquals(result.size(), 1); + Map properties = (Map) result.get("properties"); + assertEquals(properties.size(), 7); + ImmutableMap expectedURNField = + ImmutableMap.of( + "type", + "keyword", + "fields", + ImmutableMap.of( + "delimited", + ImmutableMap.of( + "type", + "text", + "analyzer", + "urn_component", + "search_analyzer", + "query_urn_component", + "search_quote_analyzer", + "quote_analyzer"), + "ngram", + ImmutableMap.of( + "type", + "search_as_you_type", + "max_shingle_size", + "4", + "doc_values", + "false", + "analyzer", + "partial_urn_component"))); + assertEquals(properties.get("urn"), expectedURNField); + assertEquals(properties.get("runId"), ImmutableMap.of("type", "keyword")); + assertTrue(properties.containsKey("editedFieldDescriptions")); + assertTrue(properties.containsKey("displayName")); + assertTrue(properties.containsKey("refEntityUrns")); + // @SearchableRef Field + Map refField = (Map) properties.get("refEntityUrns"); + assertEquals(refField.size(), 1); + Map refFieldProperty = (Map) refField.get("properties"); + + assertEquals(refFieldProperty.get("urn"), expectedURNField); + assertTrue(refFieldProperty.containsKey("displayName")); + assertTrue(refFieldProperty.containsKey("editedFieldDescriptions")); + } + + private EntityRegistry getTestEntityRegistry() { + return new ConfigEntityRegistry( + TestSearchFieldConfig.class + .getClassLoader() + .getResourceAsStream("test-entity-registry.yaml")); + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java index 38d630bc302f4e..ba13d412447952 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java @@ -35,6 +35,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.annotation.PostConstruct; import org.mockito.Mockito; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; @@ -86,6 +87,12 @@ public class SearchQueryBuilderTest extends AbstractTestNGSpringContextTests { public static final SearchQueryBuilder TEST_BUILDER = new SearchQueryBuilder(testQueryConfig, null); + @PostConstruct + public void setup() { + TEST_BUILDER.setEntityRegistry(entityRegistry); + TEST_CUSTOM_BUILDER.setEntityRegistry(entityRegistry); + } + @Test public void testQueryBuilderFulltext() { FunctionScoreQueryBuilder result = diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java index 6e2d90287d5d93..312314d431fb43 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java @@ -1,6 +1,7 @@ package com.linkedin.metadata.search.transformer; import static com.linkedin.metadata.Constants.*; +import static org.mockito.Mockito.*; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -13,11 +14,22 @@ import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.DataMapBuilder; +import com.linkedin.entity.Aspect; import com.linkedin.metadata.TestEntitySpecBuilder; import com.linkedin.metadata.TestEntityUtil; +import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.SearchableRefFieldSpec; +import com.linkedin.metadata.models.registry.ConfigEntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.elasticsearch.query.request.TestSearchFieldConfig; +import com.linkedin.r2.RemoteInvocationException; import java.io.IOException; -import java.util.Optional; +import java.net.URISyntaxException; +import java.util.*; +import org.mockito.Mockito; import org.testng.annotations.Test; public class SearchDocumentTransformerTest { @@ -132,4 +144,153 @@ public void testTransformMaxFieldValue() throws IOException { .add("123") .add("0123456789")); } + + /** + * + * + *
    + *
  • {@link SearchDocumentTransformer#setSearchableRefValue(SearchableRefFieldSpec, List, + * ObjectNode, Boolean ) } + *
+ */ + @Test + public void testSetSearchableRefValue() throws URISyntaxException, RemoteInvocationException { + AspectRetriever aspectRetriever = Mockito.mock(AspectRetriever.class); + SearchDocumentTransformer searchDocumentTransformer = + new SearchDocumentTransformer(1000, 1000, 1000); + searchDocumentTransformer.setAspectRetriever(aspectRetriever); + EntityRegistry entityRegistry = getTestEntityRegistry(); + List urnList = List.of(Urn.createFromString("urn:li:refEntity:1")); + + DataMapBuilder dataMapBuilder = new DataMapBuilder(); + dataMapBuilder.addKVPair("fieldPath", "refEntityUrn"); + dataMapBuilder.addKVPair("name", "refEntityUrnName"); + dataMapBuilder.addKVPair("description", "refEntityUrn1 description details"); + Aspect aspect = new Aspect(dataMapBuilder.convertToDataMap()); + + ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); + SearchableRefFieldSpec searchableRefFieldSpec = + entityRegistry.getEntitySpec("testRefEntity").getSearchableRefFieldSpecs().get(0); + + // Mock Behaviour + Mockito.when(aspectRetriever.getEntityRegistry()).thenReturn(entityRegistry); + Mockito.when(aspectRetriever.getLatestAspectObject(any(), anyString())).thenReturn(aspect); + + searchDocumentTransformer.setSearchableRefValue( + searchableRefFieldSpec, urnList, searchDocument, false); + assertTrue(searchDocument.has("refEntityUrns")); + assertEquals(searchDocument.get("refEntityUrns").size(), 3); + assertTrue(searchDocument.get("refEntityUrns").has("urn")); + assertTrue(searchDocument.get("refEntityUrns").has("editedFieldDescriptions")); + assertTrue(searchDocument.get("refEntityUrns").has("displayName")); + assertEquals(searchDocument.get("refEntityUrns").get("urn").asText(), "urn:li:refEntity:1"); + assertEquals( + searchDocument.get("refEntityUrns").get("editedFieldDescriptions").asText(), + "refEntityUrn1 description details"); + assertEquals( + searchDocument.get("refEntityUrns").get("displayName").asText(), "refEntityUrnName"); + } + + @Test + public void testSetSearchableRefValue_WithNonURNField() throws URISyntaxException { + AspectRetriever aspectRetriever = Mockito.mock(AspectRetriever.class); + SearchDocumentTransformer searchDocumentTransformer = + new SearchDocumentTransformer(1000, 1000, 1000); + searchDocumentTransformer.setAspectRetriever(aspectRetriever); + EntityRegistry entityRegistry = getTestEntityRegistry(); + List urnList = List.of(Urn.createFromString("urn:li:refEntity:1")); + + ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); + SearchableRefFieldSpec searchableRefFieldSpecText = + entityRegistry.getEntitySpec("testRefEntity").getSearchableRefFieldSpecs().get(1); + searchDocumentTransformer.setSearchableRefValue( + searchableRefFieldSpecText, urnList, searchDocument, false); + assertTrue(searchDocument.isEmpty()); + } + + @Test + public void testSetSearchableRefValue_RemoteInvocationException() + throws URISyntaxException, RemoteInvocationException { + AspectRetriever aspectRetriever = Mockito.mock(AspectRetriever.class); + SearchDocumentTransformer searchDocumentTransformer = + new SearchDocumentTransformer(1000, 1000, 1000); + searchDocumentTransformer.setAspectRetriever(aspectRetriever); + EntityRegistry entityRegistry = getTestEntityRegistry(); + List urnList = List.of(Urn.createFromString("urn:li:refEntity:1")); + + Mockito.when(aspectRetriever.getEntityRegistry()).thenReturn(entityRegistry); + Mockito.when( + aspectRetriever.getLatestAspectObject( + eq(Urn.createFromString("urn:li:refEntity:1")), anyString())) + .thenThrow(new RemoteInvocationException("Error")); + + ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); + SearchableRefFieldSpec searchableRefFieldSpec = + entityRegistry.getEntitySpec("testRefEntity").getSearchableRefFieldSpecs().get(0); + searchDocumentTransformer.setSearchableRefValue( + searchableRefFieldSpec, urnList, searchDocument, false); + assertTrue(searchDocument.isEmpty()); + } + + @Test + public void testSetSearchableRefValue_RemoteInvocationException_URNExist() + throws URISyntaxException, RemoteInvocationException { + AspectRetriever aspectRetriever = Mockito.mock(AspectRetriever.class); + SearchDocumentTransformer searchDocumentTransformer = + new SearchDocumentTransformer(1000, 1000, 1000); + searchDocumentTransformer.setAspectRetriever(aspectRetriever); + EntityRegistry entityRegistry = getTestEntityRegistry(); + List urnList = List.of(Urn.createFromString("urn:li:refEntity:1")); + DataMapBuilder dataMapBuilder = new DataMapBuilder(); + dataMapBuilder.addKVPair("fieldPath", "refEntityUrn"); + dataMapBuilder.addKVPair("name", "refEntityUrnName"); + dataMapBuilder.addKVPair("description", "refEntityUrn1 description details"); + + Aspect aspect = new Aspect(dataMapBuilder.convertToDataMap()); + Mockito.when(aspectRetriever.getEntityRegistry()).thenReturn(entityRegistry); + Mockito.when( + aspectRetriever.getLatestAspectObject( + eq(Urn.createFromString("urn:li:refEntity:1")), anyString())) + .thenReturn(aspect) + .thenThrow(new RemoteInvocationException("Error")); + + ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); + SearchableRefFieldSpec searchableRefFieldSpec = + entityRegistry.getEntitySpec("testRefEntity").getSearchableRefFieldSpecs().get(0); + searchDocumentTransformer.setSearchableRefValue( + searchableRefFieldSpec, urnList, searchDocument, false); + assertTrue(searchDocument.has("refEntityUrns")); + assertEquals(searchDocument.get("refEntityUrns").size(), 1); + assertTrue(searchDocument.get("refEntityUrns").has("urn")); + assertEquals(searchDocument.get("refEntityUrns").get("urn").asText(), "urn:li:refEntity:1"); + } + + @Test + void testSetSearchableRefValue_WithInvalidURN() + throws URISyntaxException, RemoteInvocationException { + AspectRetriever aspectRetriever = Mockito.mock(AspectRetriever.class); + SearchDocumentTransformer searchDocumentTransformer = + new SearchDocumentTransformer(1000, 1000, 1000); + searchDocumentTransformer.setAspectRetriever(aspectRetriever); + EntityRegistry entityRegistry = getTestEntityRegistry(); + List urnList = List.of(Urn.createFromString("urn:li:refEntity:1")); + + Mockito.when(aspectRetriever.getEntityRegistry()).thenReturn(entityRegistry); + Mockito.when(aspectRetriever.getLatestAspectObject(any(), anyString())).thenReturn(null); + SearchableRefFieldSpec searchableRefFieldSpec = + entityRegistry.getEntitySpec("testRefEntity").getSearchableRefFieldSpecs().get(0); + + ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); + searchDocumentTransformer.setSearchableRefValue( + searchableRefFieldSpec, urnList, searchDocument, false); + assertTrue(searchDocument.has("refEntityUrns")); + assertTrue(searchDocument.get("refEntityUrns").getNodeType().equals(JsonNodeType.NULL)); + } + + private EntityRegistry getTestEntityRegistry() { + return new ConfigEntityRegistry( + TestSearchFieldConfig.class + .getClassLoader() + .getResourceAsStream("test-entity-registry.yaml")); + } } diff --git a/metadata-io/src/test/resources/test-entity-registry.yaml b/metadata-io/src/test/resources/test-entity-registry.yaml new file mode 100644 index 00000000000000..e9bd46a7cf43a2 --- /dev/null +++ b/metadata-io/src/test/resources/test-entity-registry.yaml @@ -0,0 +1,10 @@ +id: test-registry +entities: + - name: testRefEntity + keyAspect: testRefEntityKey + aspects: + - testRefEntityInfo + - name: refEntity + keyAspect: refEntityKey + aspects: + - refEntityProperties \ No newline at end of file diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHook.java index 375d1580dab51e..2f64391f08ec56 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHook.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHook.java @@ -59,6 +59,7 @@ public class EntityChangeEventGeneratorHook implements MetadataChangeLogHook { Constants.EDITABLE_DATASET_PROPERTIES_ASPECT_NAME, Constants.ASSERTION_RUN_EVENT_ASPECT_NAME, Constants.DATA_PROCESS_INSTANCE_RUN_EVENT_ASPECT_NAME, + Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, // Entity Lifecycle Event Constants.DATASET_KEY_ASPECT_NAME, @@ -70,7 +71,8 @@ public class EntityChangeEventGeneratorHook implements MetadataChangeLogHook { Constants.GLOSSARY_TERM_KEY_ASPECT_NAME, Constants.DOMAIN_KEY_ASPECT_NAME, Constants.TAG_KEY_ASPECT_NAME, - Constants.STATUS_ASPECT_NAME); + Constants.STATUS_ASPECT_NAME, + Constants.BUSINESS_ATTRIBUTE_KEY_ASPECT_NAME); /** The list of change types that are supported for generating semantic change events. */ private static final Set SUPPORTED_OPERATIONS = diff --git a/metadata-jobs/pe-consumer/build.gradle b/metadata-jobs/pe-consumer/build.gradle index 2fd19af92971e2..3c9e916a96dfa6 100644 --- a/metadata-jobs/pe-consumer/build.gradle +++ b/metadata-jobs/pe-consumer/build.gradle @@ -24,6 +24,8 @@ dependencies { runtimeOnly externalDependency.logbackClassic testImplementation externalDependency.mockito testRuntimeOnly externalDependency.logbackClassic + testImplementation externalDependency.springBootTest + testImplementation externalDependency.testng } task avroSchemaSources(type: Copy) { diff --git a/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java index 46793aaaaf4a55..294301fd2b23f8 100644 --- a/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java +++ b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java @@ -3,14 +3,16 @@ import com.codahale.metrics.Histogram; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; +import com.datahub.event.hook.BusinessAttributeUpdateHook; import com.datahub.event.hook.PlatformEventHook; import com.linkedin.gms.factory.kafka.KafkaEventConsumerFactory; import com.linkedin.metadata.EventUtils; import com.linkedin.metadata.utils.metrics.MetricUtils; import com.linkedin.mxe.PlatformEvent; import com.linkedin.mxe.Topics; -import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -24,18 +26,21 @@ @Slf4j @Component @Conditional(PlatformEventProcessorCondition.class) -@Import({KafkaEventConsumerFactory.class}) +@Import({BusinessAttributeUpdateHook.class, KafkaEventConsumerFactory.class}) @EnableKafka public class PlatformEventProcessor { - private final List hooks; + @Getter private final List hooks; private final Histogram kafkaLagStats = MetricUtils.get().histogram(MetricRegistry.name(this.getClass(), "kafkaLag")); @Autowired - public PlatformEventProcessor() { + public PlatformEventProcessor(List platformEventHooks) { log.info("Creating Platform Event Processor"); - this.hooks = Collections.emptyList(); // No event hooks (yet) + this.hooks = + platformEventHooks.stream() + .filter(PlatformEventHook::isEnabled) + .collect(Collectors.toList()); this.hooks.forEach(PlatformEventHook::init); } diff --git a/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/hook/BusinessAttributeUpdateHook.java b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/hook/BusinessAttributeUpdateHook.java new file mode 100644 index 00000000000000..b5317dd0ac78c2 --- /dev/null +++ b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/hook/BusinessAttributeUpdateHook.java @@ -0,0 +1,34 @@ +package com.datahub.event.hook; + +import com.linkedin.gms.factory.common.GraphServiceFactory; +import com.linkedin.gms.factory.entity.EntityServiceFactory; +import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory; +import com.linkedin.metadata.service.BusinessAttributeUpdateHookService; +import com.linkedin.mxe.PlatformEvent; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Import; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +@Import({EntityServiceFactory.class, EntityRegistryFactory.class, GraphServiceFactory.class}) +public class BusinessAttributeUpdateHook implements PlatformEventHook { + + protected final BusinessAttributeUpdateHookService businessAttributeUpdateHookService; + + public BusinessAttributeUpdateHook( + BusinessAttributeUpdateHookService businessAttributeUpdateHookService) { + this.businessAttributeUpdateHookService = businessAttributeUpdateHookService; + } + + /** + * Invoke the hook when a PlatformEvent is received + * + * @param event + */ + @Override + public void invoke(@Nonnull PlatformEvent event) { + businessAttributeUpdateHookService.handleChangeEvent(event); + } +} diff --git a/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/hook/PlatformEventHook.java b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/hook/PlatformEventHook.java index 3083642c5bfb6f..b936656ad2b91c 100644 --- a/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/hook/PlatformEventHook.java +++ b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/hook/PlatformEventHook.java @@ -15,6 +15,14 @@ public interface PlatformEventHook { /** Initialize the hook */ default void init() {} + /** + * Return whether the hook is enabled or not. If not enabled, the below invoke method is not + * triggered + */ + default boolean isEnabled() { + return true; + } + /** Invoke the hook when a PlatformEvent is received */ void invoke(@Nonnull PlatformEvent event); } diff --git a/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java b/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java new file mode 100644 index 00000000000000..68cd2aa565b9fc --- /dev/null +++ b/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java @@ -0,0 +1,255 @@ +package com.datahub.event.hook; + +import static com.datahub.event.hook.EntityRegistryTestUtil.ENTITY_REGISTRY; +import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_ASPECT; +import static com.linkedin.metadata.Constants.SCHEMA_FIELD_ENTITY_NAME; +import static com.linkedin.metadata.search.utils.QueryUtils.EMPTY_FILTER; +import static com.linkedin.metadata.search.utils.QueryUtils.newFilter; +import static com.linkedin.metadata.search.utils.QueryUtils.newRelationshipFilter; +import static org.mockito.ArgumentMatchers.eq; +import static org.testng.Assert.assertEquals; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.linkedin.businessattribute.BusinessAttributes; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.GlobalTags; +import com.linkedin.common.TagAssociation; +import com.linkedin.common.TagAssociationArray; +import com.linkedin.common.urn.TagUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.DataMap; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.graph.GraphService; +import com.linkedin.metadata.graph.RelatedEntitiesResult; +import com.linkedin.metadata.graph.RelatedEntity; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import com.linkedin.metadata.service.BusinessAttributeUpdateHookService; +import com.linkedin.metadata.timeline.data.ChangeCategory; +import com.linkedin.metadata.timeline.data.ChangeOperation; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.PlatformEvent; +import com.linkedin.mxe.PlatformEventHeader; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.platform.event.v1.EntityChangeEvent; +import com.linkedin.platform.event.v1.Parameters; +import com.linkedin.util.Pair; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.Future; +import org.mockito.Mockito; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class BusinessAttributeUpdateHookTest { + + private static final String TEST_BUSINESS_ATTRIBUTE_URN = + "urn:li:businessAttribute:12668aea-009b-400e-8408-e661c3a230dd"; + private static final String BUSINESS_ATTRIBUTE_OF = "BusinessAttributeOf"; + private static final Urn SCHEMA_FIELD_URN = + UrnUtils.getUrn( + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD),field_bar)"); + private static final String TAG_NAME = "test"; + private static final long EVENT_TIME = 123L; + private static final String TEST_ACTOR_URN = "urn:li:corpuser:test"; + private static Urn actorUrn; + private GraphService mockGraphService; + private EntityService mockEntityService; + private BusinessAttributeUpdateHook businessAttributeUpdateHook; + private BusinessAttributeUpdateHookService businessAttributeServiceHook; + + @BeforeMethod + public void setupTest() throws URISyntaxException { + mockGraphService = Mockito.mock(GraphService.class); + mockEntityService = Mockito.mock(EntityService.class); + actorUrn = Urn.createFromString(TEST_ACTOR_URN); + businessAttributeServiceHook = + new BusinessAttributeUpdateHookService( + mockGraphService, mockEntityService, ENTITY_REGISTRY, 100); + businessAttributeUpdateHook = new BusinessAttributeUpdateHook(businessAttributeServiceHook); + } + + @Test + public void testMCLOnBusinessAttributeUpdate() throws Exception { + PlatformEvent platformEvent = createPlatformEventBusinessAttribute(); + final RelatedEntitiesResult mockRelatedEntities = + new RelatedEntitiesResult( + 0, + 1, + 1, + ImmutableList.of( + new RelatedEntity(BUSINESS_ATTRIBUTE_OF, SCHEMA_FIELD_URN.toString()))); + // mock response + Mockito.when( + mockGraphService.findRelatedEntities( + null, + newFilter("urn", TEST_BUSINESS_ATTRIBUTE_URN), + null, + EMPTY_FILTER, + Arrays.asList(BUSINESS_ATTRIBUTE_OF), + newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.INCOMING), + 0, + 100)) + .thenReturn(mockRelatedEntities); + assertEquals(mockRelatedEntities.getTotal(), 1); + + Mockito.when( + mockEntityService.getLatestEnvelopedAspect( + eq(SCHEMA_FIELD_ENTITY_NAME), eq(SCHEMA_FIELD_URN), eq(BUSINESS_ATTRIBUTE_ASPECT))) + .thenReturn(envelopedAspect()); + + // mock response + Mockito.when( + mockEntityService.alwaysProduceMCLAsync( + Mockito.any(Urn.class), + Mockito.anyString(), + Mockito.anyString(), + Mockito.any(AspectSpec.class), + eq(null), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(ChangeType.class))) + .thenReturn(Pair.of(Mockito.mock(Future.class), false)); + + // invoke + businessAttributeServiceHook.handleChangeEvent(platformEvent); + + // verify + Mockito.verify(mockGraphService, Mockito.times(1)) + .findRelatedEntities( + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.anyInt(), + Mockito.anyInt()); + + Mockito.verify(mockEntityService, Mockito.times(1)) + .alwaysProduceMCLAsync( + Mockito.any(Urn.class), + Mockito.anyString(), + Mockito.anyString(), + Mockito.any(AspectSpec.class), + eq(null), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(ChangeType.class)); + } + + @Test + private void testMCLOnInvalidCategory() throws Exception { + PlatformEvent platformEvent = createPlatformEventInvalidCategory(); + + // invoke + businessAttributeServiceHook.handleChangeEvent(platformEvent); + + // verify + Mockito.verify(mockGraphService, Mockito.times(0)) + .findRelatedEntities( + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.anyInt(), + Mockito.anyInt()); + + Mockito.verify(mockEntityService, Mockito.times(0)) + .alwaysProduceMCLAsync( + Mockito.any(Urn.class), + Mockito.anyString(), + Mockito.anyString(), + Mockito.any(AspectSpec.class), + eq(null), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(ChangeType.class)); + } + + public static PlatformEvent createPlatformEventBusinessAttribute() throws Exception { + final GlobalTags newTags = new GlobalTags(); + final TagUrn newTagUrn = new TagUrn(TAG_NAME); + newTags.setTags( + new TagAssociationArray(ImmutableList.of(new TagAssociation().setTag(newTagUrn)))); + PlatformEvent platformEvent = + createChangeEvent( + Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME, + Urn.createFromString(TEST_BUSINESS_ATTRIBUTE_URN), + ChangeCategory.TAG, + ChangeOperation.ADD, + newTagUrn.toString(), + ImmutableMap.of("tagUrn", newTagUrn.toString()), + actorUrn); + return platformEvent; + } + + public static PlatformEvent createPlatformEventInvalidCategory() throws Exception { + final GlobalTags newTags = new GlobalTags(); + final TagUrn newTagUrn = new TagUrn(TAG_NAME); + newTags.setTags( + new TagAssociationArray(ImmutableList.of(new TagAssociation().setTag(newTagUrn)))); + PlatformEvent platformEvent = + createChangeEvent( + Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME, + Urn.createFromString(TEST_BUSINESS_ATTRIBUTE_URN), + ChangeCategory.DOMAIN, + ChangeOperation.ADD, + newTagUrn.toString(), + ImmutableMap.of("tagUrn", newTagUrn.toString()), + actorUrn); + return platformEvent; + } + + private static PlatformEvent createChangeEvent( + String entityType, + Urn entityUrn, + ChangeCategory category, + ChangeOperation operation, + String modifier, + Map parameters, + Urn actor) { + final EntityChangeEvent changeEvent = new EntityChangeEvent(); + changeEvent.setEntityType(entityType); + changeEvent.setEntityUrn(entityUrn); + changeEvent.setCategory(category.name()); + changeEvent.setOperation(operation.name()); + if (modifier != null) { + changeEvent.setModifier(modifier); + } + changeEvent.setAuditStamp( + new AuditStamp().setActor(actor).setTime(BusinessAttributeUpdateHookTest.EVENT_TIME)); + changeEvent.setVersion(0); + if (parameters != null) { + changeEvent.setParameters(new Parameters(new DataMap(parameters))); + } + final PlatformEvent platformEvent = new PlatformEvent(); + platformEvent.setName(Constants.CHANGE_EVENT_PLATFORM_EVENT_NAME); + platformEvent.setHeader( + new PlatformEventHeader().setTimestampMillis(BusinessAttributeUpdateHookTest.EVENT_TIME)); + platformEvent.setPayload(GenericRecordUtils.serializePayload(changeEvent)); + return platformEvent; + } + + private EnvelopedAspect envelopedAspect() { + EnvelopedAspect envelopedAspect = new EnvelopedAspect(); + envelopedAspect.setValue(new Aspect(new BusinessAttributes().data())); + envelopedAspect.setSystemMetadata(new SystemMetadata()); + return envelopedAspect; + } +} diff --git a/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/EntityRegistryTestUtil.java b/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/EntityRegistryTestUtil.java new file mode 100644 index 00000000000000..62f6fd0fceda24 --- /dev/null +++ b/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/EntityRegistryTestUtil.java @@ -0,0 +1,22 @@ +package com.datahub.event.hook; + +import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor; +import com.linkedin.metadata.models.registry.ConfigEntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistry; + +public class EntityRegistryTestUtil { + private EntityRegistryTestUtil() {} + + public static final EntityRegistry ENTITY_REGISTRY; + + static { + EntityRegistryTestUtil.class + .getClassLoader() + .setClassAssertionStatus(PathSpecBasedSchemaAnnotationVisitor.class.getName(), false); + ENTITY_REGISTRY = + new ConfigEntityRegistry( + EntityRegistryTestUtil.class + .getClassLoader() + .getResourceAsStream("test-entity-registry.yml")); + } +} diff --git a/metadata-jobs/pe-consumer/src/test/resources/test-entity-registry.yml b/metadata-jobs/pe-consumer/src/test/resources/test-entity-registry.yml new file mode 100644 index 00000000000000..f7296ec240750c --- /dev/null +++ b/metadata-jobs/pe-consumer/src/test/resources/test-entity-registry.yml @@ -0,0 +1,12 @@ +entities: + - name: dataset + keyAspect: datasetKey + aspects: + - editableSchemaMetadata + - name: schemaField + category: core + keyAspect: schemaFieldKey + aspects: + - businessAttributes +events: + - name: entityChangeEvent diff --git a/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeAssociation.pdl new file mode 100644 index 00000000000000..5422864185f141 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeAssociation.pdl @@ -0,0 +1,9 @@ +namespace com.linkedin.businessattribute +import com.linkedin.common.BusinessAttributeUrn + +record BusinessAttributeAssociation { + /** + * Urn of the applied businessAttribute + */ + businessAttributeUrn: BusinessAttributeUrn +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeInfo.pdl new file mode 100644 index 00000000000000..388164bc8ca6e4 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeInfo.pdl @@ -0,0 +1,26 @@ +namespace com.linkedin.businessattribute + +import com.linkedin.schema.SchemaFieldDataType +import com.linkedin.schema.EditableSchemaFieldInfo +import com.linkedin.common.CustomProperties +import com.linkedin.common.ChangeAuditStamps + +/** + * Properties associated with a BusinessAttribute + */ +@Aspect = { + "name": "businessAttributeInfo" +} +record BusinessAttributeInfo includes EditableSchemaFieldInfo, CustomProperties, ChangeAuditStamps { + /** + * Display name of the BusinessAttribute + */ + @Searchable = { + "fieldType": "WORD_GRAM", + "enableAutocomplete": true, + "boostScore": 10.0, + "fieldNameAliases": [ "_entityName" ] + } + name: string + type: optional SchemaFieldDataType +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeKey.pdl new file mode 100644 index 00000000000000..5c134804af19dd --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeKey.pdl @@ -0,0 +1,14 @@ +namespace com.linkedin.businessattribute + +/** + * Key for a Query + */ +@Aspect = { + "name": "businessAttributeKey" +} +record BusinessAttributeKey { + /** + * A unique id for the Business Attribute. + */ + id: string +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributes.pdl b/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributes.pdl new file mode 100644 index 00000000000000..8b7df311d24d90 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributes.pdl @@ -0,0 +1,29 @@ +namespace com.linkedin.businessattribute + +/** + * BusinessAttribute aspect used for applying it to an entity + */ +@Aspect = { + "name": "businessAttributes" +} +record BusinessAttributes { + + /** + * Business Attribute for this field. + */ + @Relationship = { + "/businessAttributeUrn": { + "name": "BusinessAttributeOf", + "entityTypes": [ "businessAttribute" ] + } + } + @SearchableRef = { + "/businessAttributeUrn": { + "fieldName": "businessAttributeRef", + "fieldType": "URN", + "boostScore": 0.5 + "refType" : "businessAttribute" + } + } + businessAttribute: optional BusinessAttributeAssociation +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/schemafield/schemafieldInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/schemafield/schemafieldInfo.pdl new file mode 100644 index 00000000000000..086d9df34deadd --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/schemafield/schemafieldInfo.pdl @@ -0,0 +1,13 @@ +namespace com.linkedin.schemafield + +@Aspect = { + "name": "schemafieldInfo" +} + +record SchemaFieldInfo { + @Searchable = { + "fieldType": "KEYWORD", + "fieldNameAliases": [ "_entityName" ] + } + name: optional string +} \ No newline at end of file diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 55b96555e02b99..d7ab1f948b411a 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -448,8 +448,10 @@ entities: category: core keyAspect: schemaFieldKey aspects: + - schemafieldInfo - structuredProperties - forms + - businessAttributes - name: globalSettings doc: Global settings for an the platform category: internal @@ -522,6 +524,14 @@ entities: aspects: - ownershipTypeInfo - status + - name: businessAttribute + category: core + keyAspect: businessAttributeKey + aspects: + - businessAttributeInfo + - status + - ownership + - institutionalMemory - name: dataContract category: core keyAspect: dataContractKey diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml index 4d50412da6ea52..414ade52a71d68 100644 --- a/metadata-service/configuration/src/main/resources/application.yml +++ b/metadata-service/configuration/src/main/resources/application.yml @@ -369,6 +369,7 @@ featureFlags: erModelRelationshipFeatureEnabled: ${ER_MODEL_RELATIONSHIP_FEATURE_ENABLED:false} # Enable Join Tables Feature and show within Dataset view as Relations nestedDomainsEnabled: ${NESTED_DOMAINS_ENABLED:true} # Enables the nested Domains feature that allows users to have sub-Domains. If this is off, Domains appear "flat" again schemaFieldEntityFetchEnabled: ${SCHEMA_FIELD_ENTITY_FETCH_ENABLED:true} # Enables fetching for schema field entities from the database when we hydrate them on schema fields + businessAttributeEntityEnabled: ${BUSINESS_ATTRIBUTE_ENTITY_ENABLED:false} # Enables business attribute entity which can be associated with field of dataset entityChangeEvents: enabled: ${ENABLE_ENTITY_CHANGE_EVENTS_HOOK:true} @@ -442,3 +443,6 @@ springdoc.api-docs.groups.enabled: true forms: hook: enabled: { $FORMS_HOOK_ENABLED:true } + +businessAttribute: + fetchRelatedEntitiesCount: ${BUSINESS_ATTRIBUTE_RELATED_ENTITIES_COUNT:100000} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/businessattribute/BusinessAttributeServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/businessattribute/BusinessAttributeServiceFactory.java new file mode 100644 index 00000000000000..8c7b5f2a83854c --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/businessattribute/BusinessAttributeServiceFactory.java @@ -0,0 +1,20 @@ +package com.linkedin.gms.factory.businessattribute; + +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.service.BusinessAttributeService; +import javax.annotation.Nonnull; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +@Component +public class BusinessAttributeServiceFactory { + @Bean(name = "businessAttributeService") + @Scope("singleton") + @Nonnull + protected BusinessAttributeService getINSTANCE( + @Qualifier("entityClient") final EntityClient entityClient) throws Exception { + return new BusinessAttributeService(entityClient); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java index d6d5e85cc3f380..ee5e6629572615 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java @@ -26,6 +26,7 @@ import com.linkedin.metadata.graph.SiblingGraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.recommendation.RecommendationsService; +import com.linkedin.metadata.service.BusinessAttributeService; import com.linkedin.metadata.service.DataProductService; import com.linkedin.metadata.service.ERModelRelationshipService; import com.linkedin.metadata.service.FormService; @@ -180,6 +181,10 @@ public class GraphQLEngineFactory { @Value("${platformAnalytics.enabled}") // TODO: Migrate to DATAHUB_ANALYTICS_ENABLED private Boolean isAnalyticsEnabled; + @Autowired + @Qualifier("businessAttributeService") + private BusinessAttributeService businessAttributeService; + @Bean(name = "graphQLEngine") @Nonnull protected GraphQLEngine graphQLEngine( @@ -229,6 +234,7 @@ protected GraphQLEngine graphQLEngine( args.setGraphQLQueryComplexityLimit( configProvider.getGraphQL().getQuery().getComplexityLimit()); args.setGraphQLQueryDepthLimit(configProvider.getGraphQL().getQuery().getDepthLimit()); + args.setBusinessAttributeService(businessAttributeService); return new GmsGraphQLEngine(args).builder().build(); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/eventgenerator/EntityChangeEventGeneratorRegistryFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/eventgenerator/EntityChangeEventGeneratorRegistryFactory.java index 50d4125257fb26..7d0c291ecd7ebb 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/eventgenerator/EntityChangeEventGeneratorRegistryFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/eventgenerator/EntityChangeEventGeneratorRegistryFactory.java @@ -4,6 +4,8 @@ import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.timeline.eventgenerator.AssertionRunEventChangeEventGenerator; +import com.linkedin.metadata.timeline.eventgenerator.BusinessAttributeAssociationChangeEventGenerator; +import com.linkedin.metadata.timeline.eventgenerator.BusinessAttributeInfoChangeEventGenerator; import com.linkedin.metadata.timeline.eventgenerator.DataProcessInstanceRunEventChangeEventGenerator; import com.linkedin.metadata.timeline.eventgenerator.DatasetPropertiesChangeEventGenerator; import com.linkedin.metadata.timeline.eventgenerator.DeprecationChangeEventGenerator; @@ -51,6 +53,10 @@ protected EntityChangeEventGeneratorRegistry entityChangeEventGeneratorRegistry( registry.register( EDITABLE_DATASET_PROPERTIES_ASPECT_NAME, new EditableDatasetPropertiesChangeEventGenerator()); + registry.register( + BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME, new BusinessAttributeInfoChangeEventGenerator()); + registry.register( + BUSINESS_ATTRIBUTE_ASSOCIATION, new BusinessAttributeAssociationChangeEventGenerator()); // Entity Lifecycle Differs registry.register(DATASET_KEY_ASPECT_NAME, new EntityKeyChangeEventGenerator<>()); @@ -65,6 +71,7 @@ protected EntityChangeEventGeneratorRegistry entityChangeEventGeneratorRegistry( registry.register(CORP_GROUP_KEY_ASPECT_NAME, new EntityKeyChangeEventGenerator<>()); registry.register(STATUS_ASPECT_NAME, new StatusChangeEventGenerator()); registry.register(DEPRECATION_ASPECT_NAME, new DeprecationChangeEventGenerator()); + registry.register(BUSINESS_ATTRIBUTE_KEY_ASPECT_NAME, new EntityKeyChangeEventGenerator<>()); // Assertion differs registry.register(ASSERTION_RUN_EVENT_ASPECT_NAME, new AssertionRunEventChangeEventGenerator()); diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataTypesStepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataTypesStepTest.java index c5539b001e9e35..9656c7d2f60efc 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataTypesStepTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataTypesStepTest.java @@ -60,7 +60,7 @@ public void testExecuteInvalidJson() throws Exception { Assert.assertThrows(RuntimeException.class, step::execute); - verify(entityService, times(1)).exists(any()); + verify(entityService, times(1)).exists(any(Collection.class)); // Verify no additional interactions verifyNoMoreInteractions(entityService); diff --git a/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/delegates/EntityApiDelegateImpl.java b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/delegates/EntityApiDelegateImpl.java new file mode 100644 index 00000000000000..8b137891791fe9 --- /dev/null +++ b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/delegates/EntityApiDelegateImpl.java @@ -0,0 +1 @@ + diff --git a/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java index f366cef4d979f9..3dfe40d6b9b4be 100644 --- a/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java +++ b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java @@ -24,6 +24,8 @@ import io.datahubproject.openapi.exception.UnauthorizedException; import io.datahubproject.openapi.generated.BrowsePathsV2AspectRequestV2; import io.datahubproject.openapi.generated.BrowsePathsV2AspectResponseV2; +import io.datahubproject.openapi.generated.BusinessAttributeInfoAspectRequestV2; +import io.datahubproject.openapi.generated.BusinessAttributeInfoAspectResponseV2; import io.datahubproject.openapi.generated.ChartInfoAspectRequestV2; import io.datahubproject.openapi.generated.ChartInfoAspectResponseV2; import io.datahubproject.openapi.generated.DataProductPropertiesAspectRequestV2; @@ -66,10 +68,12 @@ import javax.annotation.Nullable; import javax.validation.Valid; import javax.validation.constraints.Min; +import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpEntity; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +@Slf4j public class EntityApiDelegateImpl { private final OperationContext systemOperationContext; private final EntityRegistry _entityRegistry; @@ -81,6 +85,8 @@ public class EntityApiDelegateImpl { private final Class _respClazz; private final Class _scrollRespClazz; + private static final String BUSINESS_ATTRIBUTE_ERROR_MESSAGE = + "business attribute is disabled, enable it using featureflag : BUSINESS_ATTRIBUTE_ENTITY_ENABLED"; private final StackWalker walker = StackWalker.getInstance(); public EntityApiDelegateImpl( @@ -104,6 +110,9 @@ public EntityApiDelegateImpl( } public ResponseEntity get(String urn, Boolean systemMetadata, List aspects) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } String[] requestedAspects = Optional.ofNullable(aspects) .map(asp -> asp.stream().distinct().toArray(String[]::new)) @@ -128,6 +137,14 @@ public ResponseEntity> create( OpenApiEntitiesUtil.convertEntityToUpsert(b, _reqClazz, _entityRegistry) .stream()) .collect(Collectors.toList()); + + Optional aspect = aspects.stream().findFirst(); + if (aspect.isPresent()) { + String entityType = aspect.get().getEntityType(); + if (checkBusinessAttributeFlagFromEntityType(entityType)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } + } _v1Controller.postEntities(aspects, false, createIfNotExists, createEntityIfNotExists); List responses = body.stream() @@ -137,14 +154,19 @@ public ResponseEntity> create( } public ResponseEntity delete(String urn) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } _v1Controller.deleteEntities(new String[] {urn}, false, false); return new ResponseEntity<>(HttpStatus.OK); } public ResponseEntity head(String urn) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } try { Urn entityUrn = Urn.createFromString(urn); - final Authentication auth = AuthenticationContext.getAuthentication(); if (!AuthUtil.isAPIAuthorizedEntityUrns( auth, _authorizationChain, EXISTS, List.of(entityUrn))) { @@ -278,6 +300,9 @@ public ResponseEntity createOwnership( String urn, @Nullable Boolean createIfNotExists, @Nullable Boolean createEntityIfNotExists) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } String methodName = walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return createAspect( @@ -295,6 +320,9 @@ public ResponseEntity createStatus( String urn, @Nullable Boolean createIfNotExists, @Nullable Boolean createEntityIfNotExists) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } String methodName = walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return createAspect( @@ -326,12 +354,18 @@ public ResponseEntity deleteGlossaryTerms(String urn) { } public ResponseEntity deleteOwnership(String urn) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } String methodName = walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return deleteAspect(urn, methodNameToAspectName(methodName)); } public ResponseEntity deleteStatus(String urn) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } String methodName = walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return deleteAspect(urn, methodNameToAspectName(methodName)); @@ -374,6 +408,9 @@ public ResponseEntity getGlossaryTerms( public ResponseEntity getOwnership( String urn, Boolean systemMetadata) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } String methodName = walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return getAspect( @@ -385,6 +422,9 @@ public ResponseEntity getOwnership( } public ResponseEntity getStatus(String urn, Boolean systemMetadata) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } String methodName = walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return getAspect( @@ -414,12 +454,18 @@ public ResponseEntity headGlossaryTerms(String urn) { } public ResponseEntity headOwnership(String urn) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } String methodName = walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return headAspect(urn, methodNameToAspectName(methodName)); } public ResponseEntity headStatus(String urn) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } String methodName = walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return headAspect(urn, methodNameToAspectName(methodName)); @@ -624,6 +670,9 @@ public ResponseEntity createInstitutionalMe String urn, @Nullable Boolean createIfNotExists, @Nullable Boolean createEntityIfNotExists) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } String methodName = walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return createAspect( @@ -700,6 +749,9 @@ public ResponseEntity deleteEditableDatasetProperties(String urn) { } public ResponseEntity deleteInstitutionalMemory(String urn) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } String methodName = walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return deleteAspect(urn, methodNameToAspectName(methodName)); @@ -737,6 +789,9 @@ public ResponseEntity getEditableData public ResponseEntity getInstitutionalMemory( String urn, Boolean systemMetadata) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } String methodName = walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return getAspect( @@ -796,6 +851,9 @@ public ResponseEntity headEditableDatasetProperties(String urn) { } public ResponseEntity headInstitutionalMemory(String urn) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } String methodName = walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return headAspect(urn, methodNameToAspectName(methodName)); @@ -953,4 +1011,74 @@ public ResponseEntity deleteFormInfo(String urn) { walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return deleteAspect(urn, methodNameToAspectName(methodName)); } + + public ResponseEntity createBusinessAttributeInfo( + BusinessAttributeInfoAspectRequestV2 body, + String urn, + @Nullable Boolean createIfNotExists, + @Nullable Boolean createEntityIfNotExists) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return createAspect( + urn, + methodNameToAspectName(methodName), + body, + BusinessAttributeInfoAspectRequestV2.class, + BusinessAttributeInfoAspectResponseV2.class, + createIfNotExists, + createEntityIfNotExists); + } + + public ResponseEntity deleteBusinessAttributeInfo(String urn) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return deleteAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity getBusinessAttributeInfo( + String urn, Boolean systemMetadata) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return getAspect( + urn, + systemMetadata, + methodNameToAspectName(methodName), + _respClazz, + BusinessAttributeInfoAspectResponseV2.class); + } + + public ResponseEntity headBusinessAttributeInfo(String urn) { + if (checkBusinessAttributeFlagFromUrn(urn)) { + throw new UnsupportedOperationException(BUSINESS_ATTRIBUTE_ERROR_MESSAGE); + } + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return headAspect(urn, methodNameToAspectName(methodName)); + } + + private boolean checkBusinessAttributeFlagFromUrn(String urn) { + try { + return checkBusinessAttributeFlagFromEntityType(Urn.createFromString(urn).getEntityType()); + } catch (URISyntaxException e) { + return true; + } + } + + private boolean checkBusinessAttributeFlagFromEntityType(String entityType) { + return entityType.equals("businessAttribute") && !businessAttributeEntityEnabled(); + } + + private boolean businessAttributeEntityEnabled() { + return System.getenv("BUSINESS_ATTRIBUTE_ENTITY_ENABLED") != null + && Boolean.parseBoolean(System.getenv("BUSINESS_ATTRIBUTE_ENTITY_ENABLED")); + } } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java b/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java index ea86d2c0c98427..dbe1f45fae29e5 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java @@ -77,7 +77,8 @@ public enum DataHubUsageEventType { DESELECT_QUICK_FILTER_EVENT("DeselectQuickFilterEvent"), EMBED_PROFILE_VIEW_EVENT("EmbedProfileViewEvent"), EMBED_PROFILE_VIEW_IN_DATAHUB_EVENT("EmbedProfileViewInDataHubEvent"), - EMBED_LOOKUP_NOT_FOUND_EVENT("EmbedLookupNotFoundEvent"); + EMBED_LOOKUP_NOT_FOUND_EVENT("EmbedLookupNotFoundEvent"), + CREATE_BUSINESS_ATTRIBUTE("CreateBusinessAttributeEvent"); private final String type; diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java index 33dffb4ed975cb..6a3dc928a168ab 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -90,6 +90,16 @@ default boolean exists(@Nonnull Urn urn, boolean includeSoftDelete) { return exists(List.of(urn), includeSoftDelete).contains(urn); } + /** + * Returns whether the urn of the entity exists (has materialized aspects). + * + * @param urn the urn of the entity to check + * @return entities exists. + */ + default boolean exists(@Nonnull Urn urn) { + return exists(urn, true); + } + /** * Retrieves the latest aspects corresponding to a batch of {@link Urn}s based on a provided set * of aspect names. diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/BusinessAttributeService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/BusinessAttributeService.java new file mode 100644 index 00000000000000..9cb47ded0819ec --- /dev/null +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/BusinessAttributeService.java @@ -0,0 +1,39 @@ +package com.linkedin.metadata.service; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import java.util.Objects; +import java.util.Set; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class BusinessAttributeService { + private final EntityClient _entityClient; + + public BusinessAttributeService(EntityClient entityClient) { + _entityClient = entityClient; + } + + public EntityResponse getBusinessAttributeEntityResponse( + @Nonnull final Urn businessAttributeUrn, @Nonnull final Authentication authentication) { + Objects.requireNonNull(businessAttributeUrn, "business attribute must not be null"); + Objects.requireNonNull(authentication, "authentication must not be null"); + try { + return _entityClient + .batchGetV2( + Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME, + Set.of(businessAttributeUrn), + Set.of(Constants.BUSINESS_ATTRIBUTE_INFO_ASPECT_NAME), + authentication) + .get(businessAttributeUrn); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to retrieve Business Attribute with urn %s", businessAttributeUrn), + e); + } + } +} diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeCategory.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeCategory.java index 141a963c3e0145..a6b2bdc4206197 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeCategory.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeCategory.java @@ -23,7 +23,9 @@ public enum ChangeCategory { // Entity Lifecycle events (create, soft delete, hard delete) LIFECYCLE, // Run event - RUN; + RUN, + + BUSINESS_ATTRIBUTE; public static final Map, ChangeCategory> COMPOUND_CATEGORIES; diff --git a/metadata-service/war/src/main/resources/boot/data_platforms.json b/metadata-service/war/src/main/resources/boot/data_platforms.json index a3fdb595cc0797..4135280cb1ac86 100644 --- a/metadata-service/war/src/main/resources/boot/data_platforms.json +++ b/metadata-service/war/src/main/resources/boot/data_platforms.json @@ -644,5 +644,15 @@ "datasetNameDelimiter": "/", "logoUrl": "/assets/platforms/excel-logo.svg" } + }, + { + "urn": "urn:li:dataPlatform:sigma", + "aspect": { + "datasetNameDelimiter": ".", + "name": "sigma", + "displayName": "Sigma", + "type": "OTHERS", + "logoUrl": "/assets/platforms/sigmalogo.png" + } } ] diff --git a/metadata-service/war/src/main/resources/boot/policies.json b/metadata-service/war/src/main/resources/boot/policies.json index eb1f0a9b47e358..454f0ba7d1163b 100644 --- a/metadata-service/war/src/main/resources/boot/policies.json +++ b/metadata-service/war/src/main/resources/boot/policies.json @@ -32,7 +32,9 @@ "SET_WRITEABLE_PRIVILEGE", "APPLY_RETENTION_PRIVILEGE", "MANAGE_GLOBAL_OWNERSHIP_TYPES", - "GET_ANALYTICS_PRIVILEGE" + "GET_ANALYTICS_PRIVILEGE", + "CREATE_BUSINESS_ATTRIBUTE", + "MANAGE_BUSINESS_ATTRIBUTE" ], "displayName":"Root User - All Platform Privileges", "description":"Grants all platform privileges to root user.", @@ -175,7 +177,9 @@ "SET_WRITEABLE_PRIVILEGE", "APPLY_RETENTION_PRIVILEGE", "MANAGE_GLOBAL_OWNERSHIP_TYPES", - "GET_ANALYTICS_PRIVILEGE" + "GET_ANALYTICS_PRIVILEGE", + "CREATE_BUSINESS_ATTRIBUTE", + "MANAGE_BUSINESS_ATTRIBUTE" ], "displayName":"Admins - Platform Policy", "description":"Admins have all platform privileges.", @@ -214,6 +218,7 @@ "EDIT_DATASET_COL_TAGS", "EDIT_DATASET_COL_GLOSSARY_TERMS", "EDIT_DATASET_COL_DESCRIPTION", + "EDIT_DATASET_COL_BUSINESS_ATTRIBUTE_PRIVILEGE", "VIEW_DATASET_USAGE", "VIEW_DATASET_PROFILE", "EDIT_TAG_COLOR", @@ -259,7 +264,8 @@ "MANAGE_DOMAINS", "MANAGE_GLOBAL_ANNOUNCEMENTS", "MANAGE_GLOSSARIES", - "MANAGE_TAGS" + "MANAGE_TAGS", + "MANAGE_BUSINESS_ATTRIBUTE" ], "displayName":"Editors - Platform Policy", "description":"Editors can manage ingestion and view analytics.", @@ -296,6 +302,7 @@ "EDIT_DATASET_COL_TAGS", "EDIT_DATASET_COL_GLOSSARY_TERMS", "EDIT_DATASET_COL_DESCRIPTION", + "EDIT_DATASET_COL_BUSINESS_ATTRIBUTE_PRIVILEGE", "VIEW_DATASET_USAGE", "VIEW_DATASET_PROFILE", "EDIT_TAG_COLOR", @@ -445,6 +452,7 @@ "EDIT_DATASET_COL_TAGS", "EDIT_DATASET_COL_GLOSSARY_TERMS", "EDIT_DATASET_COL_DESCRIPTION", + "EDIT_DATASET_COL_BUSINESS_ATTRIBUTE_PRIVILEGE", "VIEW_DATASET_USAGE", "VIEW_DATASET_PROFILE", "EDIT_TAG_COLOR", diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java index 60b2c611440b26..6788f6e87fc0da 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java @@ -130,6 +130,18 @@ public class PoliciesConfig { "Manage Ownership Types", "Create, update and delete Ownership Types."); + public static final Privilege CREATE_BUSINESS_ATTRIBUTE_PRIVILEGE = + Privilege.of( + "CREATE_BUSINESS_ATTRIBUTE", + "Create Business Attribute", + "Create new Business Attribute."); + + public static final Privilege MANAGE_BUSINESS_ATTRIBUTE_PRIVILEGE = + Privilege.of( + "MANAGE_BUSINESS_ATTRIBUTE", + "Manage Business Attribute", + "Create, update, delete Business Attribute"); + public static final List PLATFORM_PRIVILEGES = ImmutableList.of( MANAGE_POLICIES_PRIVILEGE, @@ -150,7 +162,9 @@ public class PoliciesConfig { CREATE_DOMAINS_PRIVILEGE, CREATE_GLOBAL_ANNOUNCEMENTS_PRIVILEGE, MANAGE_GLOBAL_VIEWS, - MANAGE_GLOBAL_OWNERSHIP_TYPES); + MANAGE_GLOBAL_OWNERSHIP_TYPES, + CREATE_BUSINESS_ATTRIBUTE_PRIVILEGE, + MANAGE_BUSINESS_ATTRIBUTE_PRIVILEGE); // Resource Privileges // @@ -646,6 +660,18 @@ public class PoliciesConfig { "ERModelRelationship", "update privileges for ermodelrelations", COMMON_ENTITY_PRIVILEGES); + public static final ResourcePrivileges BUSINESS_ATTRIBUTE_PRIVILEGES = + ResourcePrivileges.of( + "businessAttribute", + "Business Attribute", + "Business Attribute created on Datahub", + ImmutableList.of( + VIEW_ENTITY_PAGE_PRIVILEGE, + EDIT_ENTITY_OWNERS_PRIVILEGE, + EDIT_ENTITY_DOCS_PRIVILEGE, + EDIT_ENTITY_TAGS_PRIVILEGE, + EDIT_ENTITY_GLOSSARY_TERMS_PRIVILEGE)); + public static final List ENTITY_RESOURCE_PRIVILEGES = ImmutableList.of( DATASET_PRIVILEGES, @@ -662,7 +688,8 @@ public class PoliciesConfig { CORP_USER_PRIVILEGES, NOTEBOOK_PRIVILEGES, DATA_PRODUCT_PRIVILEGES, - ER_MODEL_RELATIONSHIP_PRIVILEGES); + ER_MODEL_RELATIONSHIP_PRIVILEGES, + BUSINESS_ATTRIBUTE_PRIVILEGES); // Merge all entity specific resource privileges to create a superset of all resource privileges public static final ResourcePrivileges ALL_RESOURCE_PRIVILEGES = diff --git a/mock-entity-registry/src/main/java/mock/MockAspectSpec.java b/mock-entity-registry/src/main/java/mock/MockAspectSpec.java index 92321cce3d9053..8be6f83832abcd 100644 --- a/mock-entity-registry/src/main/java/mock/MockAspectSpec.java +++ b/mock-entity-registry/src/main/java/mock/MockAspectSpec.java @@ -6,6 +6,7 @@ import com.linkedin.metadata.models.RelationshipFieldSpec; import com.linkedin.metadata.models.SearchScoreFieldSpec; import com.linkedin.metadata.models.SearchableFieldSpec; +import com.linkedin.metadata.models.SearchableRefFieldSpec; import com.linkedin.metadata.models.TimeseriesFieldCollectionSpec; import com.linkedin.metadata.models.TimeseriesFieldSpec; import com.linkedin.metadata.models.annotation.AspectAnnotation; @@ -20,6 +21,7 @@ public MockAspectSpec( @Nonnull List relationshipFieldSpecs, @Nonnull List timeseriesFieldSpecs, @Nonnull List timeseriesFieldCollectionSpecs, + @Nonnull final List searchableRefFieldSpecs, RecordDataSchema schema, Class aspectClass) { super( @@ -29,6 +31,7 @@ public MockAspectSpec( relationshipFieldSpecs, timeseriesFieldSpecs, timeseriesFieldCollectionSpecs, + searchableRefFieldSpecs, schema, aspectClass); } diff --git a/mock-entity-registry/src/main/java/mock/MockEntitySpec.java b/mock-entity-registry/src/main/java/mock/MockEntitySpec.java index 0013d6615a71d4..f34faea89a870d 100644 --- a/mock-entity-registry/src/main/java/mock/MockEntitySpec.java +++ b/mock-entity-registry/src/main/java/mock/MockEntitySpec.java @@ -89,6 +89,7 @@ public AspectSpec createAspectSpec(T type, String nam Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), type.schema(), (Class) type.getClass().asSubclass(RecordTemplate.class)); } diff --git a/smoke-test/tests/cypress/cypress/e2e/businessAttribute/attribute_mutations.js b/smoke-test/tests/cypress/cypress/e2e/businessAttribute/attribute_mutations.js new file mode 100644 index 00000000000000..decee024f050b0 --- /dev/null +++ b/smoke-test/tests/cypress/cypress/e2e/businessAttribute/attribute_mutations.js @@ -0,0 +1,119 @@ +import { aliasQuery, hasOperationName } from "../utils"; + +describe("attribute list adding tags and terms", () => { + let businessAttributeEntityEnabled; + + beforeEach(() => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + aliasQuery(req, "appConfig"); + }); + }); + + const setBusinessAttributeFeatureFlag = () => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + if (hasOperationName(req, "appConfig")) { + req.reply((res) => { + businessAttributeEntityEnabled = res.body.data.appConfig.featureFlags.businessAttributeEntityEnabled; + return res; + }); + } + }).as('apiCall'); + }; + + + it("can create and add a tag to business attribute and visit new tag page", () => { + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit("/business-attribute"); + cy.wait('@apiCall').then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(3000); + cy.waitTextVisible("Business Attribute"); + cy.wait(3000); + + cy.mouseover('[data-testid="schema-field-cypressTestAttribute-tags"]'); + cy.get('[data-testid="schema-field-cypressTestAttribute-tags"]').within(() => + cy.contains("Add Tags").click() + ); + + cy.enterTextInTestId("tag-term-modal-input", "CypressAddTagToAttribute"); + + cy.contains("Create CypressAddTagToAttribute").click({ force: true }); + + cy.get("textarea").type("CypressAddTagToAttribute Test Description"); + + cy.contains(/Create$/).click({ force: true }); + + // wait a breath for elasticsearch to index the tag being applied to the business attribute- if we navigate too quick ES + // wont know and we'll see applied to 0 entities + cy.wait(3000); + + // go to tag drawer + cy.contains("CypressAddTagToAttribute").click({ force: true }); + + cy.wait(3000); + + // Click the Tag Details to launch full profile + cy.contains("Tag Details").click({ force: true }); + + cy.wait(3000); + + // title of tag page + cy.contains("CypressAddTagToAttribute"); + + // description of tag page + cy.contains("CypressAddTagToAttribute Test Description"); + + cy.wait(3000); + // used by panel - click to search + cy.contains("1 Business Attributes").click({ force: true }); + + // verify business attribute shows up in search now + cy.contains("of 1 result").click({ force: true }); + cy.contains("cypressTestAttribute").click({ force: true }); + cy.get('[data-testid="tag-CypressAddTagToAttribute"]').within(() => + cy.get("span[aria-label=close]").click() + ); + cy.contains("Yes").click(); + + cy.contains("CypressAddTagToAttribute").should("not.exist"); + + cy.goToTag("urn:li:tag:CypressAddTagToAttribute", "CypressAddTagToAttribute"); + cy.deleteFromDropdown(); + }); + + }); + + + it("can add and remove terms from a business attribute", () => { + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit("/business-attribute/" + "urn:li:businessAttribute:cypressTestAttribute"); + cy.wait('@apiCall').then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(3000); + cy.waitTextVisible("cypressTestAttribute"); + cy.wait(3000); + cy.clickOptionWithText("Add Terms"); + cy.selectOptionInTagTermModal("CypressTerm"); + cy.contains("CypressTerm"); + + cy.goToBusinessAttributeList(); + cy.get('[data-testid="schema-field-cypressTestAttribute-terms"]').contains("CypressTerm"); + + cy.get('[data-testid="schema-field-cypressTestAttribute-terms"]').within(() => + cy + .get("span[aria-label=close]") + .trigger("mouseover", { force: true }) + .click({ force: true }) + ); + cy.contains("Yes").click({ force: true }); + + cy.get('[data-testid="schema-field-cypressTestAttribute-terms"]').contains("CypressTerm").should("not.exist"); + }); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/businessAttribute/businessAttribute.js b/smoke-test/tests/cypress/cypress/e2e/businessAttribute/businessAttribute.js new file mode 100644 index 00000000000000..0657dc238a1541 --- /dev/null +++ b/smoke-test/tests/cypress/cypress/e2e/businessAttribute/businessAttribute.js @@ -0,0 +1,197 @@ +import { aliasQuery, hasOperationName } from "../utils"; + +describe("businessAttribute", () => { + let businessAttributeEntityEnabled; + + beforeEach(() => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + aliasQuery(req, "appConfig"); + }); + }); + + const setBusinessAttributeFeatureFlag = () => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + if (hasOperationName(req, "appConfig")) { + req.reply((res) => { + businessAttributeEntityEnabled = res.body.data.appConfig.featureFlags.businessAttributeEntityEnabled; + return res; + }); + } + }).as('apiCall'); + }; + + it('go to business attribute page, create attribute ', function () { + const urn="urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"; + const businessAttribute="CypressBusinessAttribute"; + const datasetName = "cypress_logging_events"; + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit("/business-attribute"); + cy.wait('@apiCall').then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(3000); + cy.waitTextVisible("Business Attribute"); + cy.wait(3000); + cy.clickOptionWithText("Create Business Attribute"); + cy.addBusinessAttributeViaModal(businessAttribute, "Create Business Attribute", businessAttribute, "create-business-attribute-button"); + + cy.wait(3000); + cy.goToBusinessAttributeList() + + cy.wait(3000) + cy.contains(businessAttribute).should("be.visible"); + + cy.addAttributeToDataset(urn, datasetName, businessAttribute); + + cy.get('[data-testid="schema-field-event_name-businessAttribute"]').within(() => + cy + .get("span[aria-label=close]") + .trigger("mouseover", { force: true }) + .click({ force: true }) + ); + cy.contains("Yes").click({ force: true }); + + cy.get('[data-testid="schema-field-event_name-businessAttribute"]').contains("CypressBusinessAttribute").should("not.exist"); + + cy.goToBusinessAttributeList(); + cy.clickOptionWithText(businessAttribute); + cy.deleteFromDropdown(); + + cy.goToBusinessAttributeList(); + cy.ensureTextNotPresent(businessAttribute); + }); + }); + + it('Inheriting tags and terms from business attribute to dataset ', function () { + const urn="urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"; + const businessAttribute="CypressAttribute"; + const datasetName = "cypress_logging_events"; + const term="CypressTerm"; + const tag="Cypress"; + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit("/dataset/" + urn); + cy.wait('@apiCall').then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(5000); + cy.waitTextVisible(datasetName); + cy.clickOptionWithText("event_name"); + cy.contains("Business Attribute"); + cy.get('[data-testid="schema-field-event_name-businessAttribute"]').within(() => + cy.contains("Add Attribute").click() + ); + cy.selectOptionInAttributeModal(businessAttribute); + cy.contains(businessAttribute); + cy.contains(term); + cy.contains(tag); + }); + }); + + it("can visit related entities", () => { + const businessAttribute="CypressAttribute"; + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit("/business-attribute"); + cy.wait('@apiCall').then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(3000); + cy.waitTextVisible("Business Attribute"); + cy.wait(3000); + cy.clickOptionWithText(businessAttribute); + cy.clickOptionWithText("Related Entities"); + //cy.visit("/business-attribute/urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449/Related%20Entities"); + //cy.wait(5000); + cy.contains("of 0").should("not.exist"); + cy.contains(/of [0-9]+/); + }); + }); + + + it("can search related entities by query", () => { + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit("/business-attribute/urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449/Related%20Entities"); + cy.wait('@apiCall').then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.get('[placeholder="Filter entities..."]').click().type( + "event_n{enter}" + ); + cy.wait(5000); + cy.contains("of 0").should("not.exist"); + cy.contains(/of 1/); + cy.contains("event_name"); + }); + }); + + it("remove business attribute from dataset", () => { + const urn="urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"; + const datasetName = "cypress_logging_events"; + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit("/dataset/" + urn); + cy.wait('@apiCall').then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(5000); + cy.waitTextVisible(datasetName); + + cy.wait(3000); + cy.get('body').then(($body) => { + if ($body.find('button[aria-label="Close"]').length > 0) { + cy.get('button[aria-label="Close"]').click(); + } + }); + cy.clickOptionWithText("event_name"); + cy.get('[data-testid="schema-field-event_name-businessAttribute"]').within(() => + cy + .get("span[aria-label=close]") + .trigger("mouseover", { force: true }) + .click({ force: true }) + ); + cy.contains("Yes").click({ force: true }); + + cy.get('[data-testid="schema-field-event_name-businessAttribute"]').contains("CypressAttribute").should("not.exist"); + }); + }); + + it("update the data type of a business attribute", () => { + const businessAttribute="cypressTestAttribute"; + setBusinessAttributeFeatureFlag(); + cy.login(); + cy.visit("/business-attribute"); + cy.wait('@apiCall').then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(3000); + cy.waitTextVisible("Business Attribute"); + cy.wait(3000); + + cy.clickOptionWithText(businessAttribute); + + cy.get('[data-testid="edit-data-type-button"]').within(() => + cy + .get("span[aria-label=edit]") + .trigger("mouseover", { force: true }) + .click({ force: true }) + ); + + cy.get('[data-testid="add-data-type-option"]').get('.ant-select-selection-search-input').click({multiple: true}); + + cy.get('.ant-select-item-option-content') + .contains('STRING') + .click(); + + cy.contains("STRING"); + }); + }); +}); diff --git a/smoke-test/tests/cypress/cypress/e2e/home/home.js b/smoke-test/tests/cypress/cypress/e2e/home/home.js index 8fa6b43e5b5d21..05140486e189b6 100644 --- a/smoke-test/tests/cypress/cypress/e2e/home/home.js +++ b/smoke-test/tests/cypress/cypress/e2e/home/home.js @@ -1,12 +1,39 @@ +import { aliasQuery, hasOperationName } from "../utils"; + describe('home', () => { + let businessAttributeEntityEnabled; + + beforeEach(() => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + aliasQuery(req, "appConfig"); + }); + }); + + const setBusinessAttributeFeatureFlag = () => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + if (hasOperationName(req, "appConfig")) { + req.reply((res) => { + businessAttributeEntityEnabled = res.body.data.appConfig.featureFlags.businessAttributeEntityEnabled; + return res; + }); + } + }).as('apiCall'); + }; it('home page shows ', () => { + setBusinessAttributeFeatureFlag(); cy.login(); cy.visit('/'); - cy.get('img[src="/assets/platforms/datahublogo.png"]').should('exist'); + // cy.get('img[src="/assets/platforms/datahublogo.png"]').should('exist'); cy.get('[data-testid="entity-type-browse-card-DATASET"]').should('exist'); cy.get('[data-testid="entity-type-browse-card-DASHBOARD"]').should('exist'); cy.get('[data-testid="entity-type-browse-card-CHART"]').should('exist'); cy.get('[data-testid="entity-type-browse-card-DATA_FLOW"]').should('exist'); cy.get('[data-testid="entity-type-browse-card-GLOSSARY_TERM"]').should('exist'); + cy.wait('@apiCall').then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.get('[data-testid="entity-type-browse-card-BUSINESS_ATTRIBUTE"]').should('exist'); + }); }); - }) \ No newline at end of file + }) diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/mutations.js b/smoke-test/tests/cypress/cypress/e2e/mutations/mutations.js index 7f8a4e4f8f335c..e2a74a15d3dfcf 100644 --- a/smoke-test/tests/cypress/cypress/e2e/mutations/mutations.js +++ b/smoke-test/tests/cypress/cypress/e2e/mutations/mutations.js @@ -1,16 +1,52 @@ +import { aliasQuery, hasOperationName } from "../utils"; + describe("mutations", () => { + let businessAttributeEntityEnabled; + + beforeEach(() => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + aliasQuery(req, "appConfig"); + }); + }); + + const setBusinessAttributeFeatureFlag = () => { + cy.intercept("POST", "/api/v2/graphql", (req) => { + if (hasOperationName(req, "appConfig")) { + req.reply((res) => { + businessAttributeEntityEnabled = res.body.data.appConfig.featureFlags.businessAttributeEntityEnabled; + return res; + }); + } + }).as('apiCall'); + }; + before(() => { // warm up elastic by issuing a `*` search cy.login(); - cy.goToStarSearchList(); - cy.wait(5000); + //Commented below function, and used individual commands below with wait + // cy.goToStarSearchList(); + cy.visit("/search?query=%2A"); + cy.wait(3000) + cy.waitTextVisible("Showing") + cy.waitTextVisible("results") + cy.wait(2000); + cy.get('body').then(($body) => { + if ($body.find('button[aria-label="Close"]').length > 0) { + cy.get('button[aria-label="Close"]').click(); + } + }); + cy.wait(2000); }); it("can create and add a tag to dataset and visit new tag page", () => { - cy.deleteUrn("urn:li:tag:CypressTestAddTag"); + // cy.deleteUrn("urn:li:tag:CypressTestAddTag"); cy.login(); cy.goToDataset("urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)", "cypress_logging_events"); - + cy.get('body').then(($body) => { + if ($body.find('button[aria-label="Close"]').length > 0) { + cy.get('button[aria-label="Close"]').click(); + } + }); cy.contains("Add Tag").click({ force: true }); cy.enterTextInTestId("tag-term-modal-input", "CypressTestAddTag"); @@ -28,21 +64,23 @@ describe("mutations", () => { // go to tag drawer cy.contains("CypressTestAddTag").click({ force: true }); - cy.wait(1000); + cy.wait(2000); // Click the Tag Details to launch full profile cy.contains("Tag Details").click({ force: true }); - cy.wait(1000); + cy.wait(2000); // title of tag page cy.contains("CypressTestAddTag"); + cy.wait(2000); // description of tag page cy.contains("CypressTestAddTag Test Description"); // used by panel - click to search - cy.contains("1 Datasets").click({ force: true }); + cy.wait(3000); + cy.contains("1 Datasets").click({ force: true }); // verify dataset shows up in search now cy.contains("of 1 result").click({ force: true }); @@ -53,8 +91,7 @@ describe("mutations", () => { cy.contains("Yes").click(); cy.contains("CypressTestAddTag").should("not.exist"); - - cy.deleteUrn("urn:li:tag:CypressTestAddTag"); + // cy.deleteUrn("urn:li:tag:CypressTestAddTag"); }); it("can add and remove terms from a dataset", () => { @@ -97,12 +134,12 @@ describe("mutations", () => { // go to tag drawer cy.contains("CypressTestAddTag2").click({ force: true }); - cy.wait(1000); + cy.wait(2000); // Click the Tag Details to launch full profile cy.contains("Tag Details").click({ force: true }); - cy.wait(1000); + cy.wait(2000); // title of tag page cy.contains("CypressTestAddTag2"); @@ -111,6 +148,7 @@ describe("mutations", () => { cy.contains("CypressTestAddTag2 Test Description"); // used by panel - click to search + cy.wait(3000); cy.contains("1 Datasets").click(); // verify dataset shows up in search now @@ -127,7 +165,7 @@ describe("mutations", () => { cy.contains("CypressTestAddTag2").should("not.exist"); - cy.deleteUrn("urn:li:tag:CypressTestAddTag2"); + // cy.deleteUrn("urn:li:tag:CypressTestAddTag2"); }); it("can add and remove terms from a dataset field", () => { @@ -154,4 +192,42 @@ describe("mutations", () => { cy.contains("CypressTerm").should("not.exist"); }); + + it("can add and remove business attribute from a dataset field", () => { + setBusinessAttributeFeatureFlag(); + cy.login(); + // make space for the glossary term column + cy.viewport(2000, 800); + cy.visit("/dataset/" + "urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)"); + cy.wait('@apiCall').then(() => { + if (!businessAttributeEntityEnabled) { + return; + } + cy.wait(5000); + cy.waitTextVisible("cypress_logging_events"); + cy.clickOptionWithText("event_data"); + cy.get('[data-testid="schema-field-event_data-businessAttribute"]').trigger( + "mouseover", + { force: true } + ); + cy.get('[data-testid="schema-field-event_data-businessAttribute"]').within(() => + cy.contains("Add Attribute").click({ force: true }) + ); + + cy.selectOptionInAttributeModal("cypressTestAttribute"); + cy.wait(2000); + cy.contains("cypressTestAttribute"); + + cy.get('[data-testid="schema-field-event_data-businessAttribute"]'). + within(() => + cy + .get("span[aria-label=close]") + .trigger("mouseover", { force: true }) + .click({ force: true }) + ); + cy.contains("Yes").click({ force: true }); + + cy.contains("cypressTestAttribute").should("not.exist"); + }); + }); }); diff --git a/smoke-test/tests/cypress/cypress/support/commands.js b/smoke-test/tests/cypress/cypress/support/commands.js index 3f3a8fe94f962f..c670e1b5732450 100644 --- a/smoke-test/tests/cypress/cypress/support/commands.js +++ b/smoke-test/tests/cypress/cypress/support/commands.js @@ -68,6 +68,12 @@ Cypress.Commands.add("goToGlossaryList", () => { cy.waitTextVisible("Glossary"); }); +Cypress.Commands.add("goToBusinessAttributeList", () => { + cy.visit("/business-attribute"); + cy.waitTextVisible("Business Attribute"); + cy.wait(3000); +}); + Cypress.Commands.add("goToDomainList", () => { cy.visit("/domains"); cy.waitTextVisible("Domains"); @@ -104,9 +110,26 @@ Cypress.Commands.add("goToDataset", (urn, dataset_name) => { cy.visit( "/dataset/" + urn ); + cy.wait(5000); cy.waitTextVisible(dataset_name); }); +Cypress.Commands.add("goToBusinessAttribute", (urn, attribute_name) => { + cy.visit( + "/business-attribute/" + urn + ); + cy.wait(5000); + cy.waitTextVisible(attribute_name); +}); + +Cypress.Commands.add("goToTag", (urn, tag_name) => { + cy.visit( + "/tag/" + urn + ); + cy.wait(5000); + cy.waitTextVisible(tag_name); +}); + Cypress.Commands.add("goToEntityLineageGraph", (entity_type, urn) => { cy.visit( `/${entity_type}/${urn}?is_lineage_mode=true` @@ -198,6 +221,14 @@ Cypress.Commands.add("addViaModal", (text, modelHeader, value, dataTestId) => { cy.contains(value).should('be.visible'); }); +Cypress.Commands.add("addBusinessAttributeViaModal", (text, modelHeader, value, dataTestId) => { + cy.waitTextVisible(modelHeader); + cy.get(".ant-input-affix-wrapper > input[type='text']").first().type(text); + cy.get('[data-testid="' + dataTestId + '"]').click(); + cy.wait(3000); + cy.contains(value).should('be.visible'); +}); + Cypress.Commands.add("ensureTextNotPresent", (text) => { cy.contains(text).should("not.exist"); }); @@ -286,6 +317,24 @@ Cypress.Commands.add('addTermToDataset', (urn, dataset_name, term) => { cy.contains(term); }); +Cypress.Commands.add('addTermToBusinessAttribute', (urn, attribute_name, term) => { + cy.goToBusinessAttribute(urn, attribute_name); + cy.clickOptionWithText("Add Terms"); + cy.selectOptionInTagTermModal(term); + cy.contains(term); +}); + +Cypress.Commands.add('addAttributeToDataset', (urn, dataset_name, businessAttribute) => { + cy.goToDataset(urn, dataset_name); + cy.clickOptionWithText("event_name"); + cy.contains("Business Attribute"); + cy.get('[data-testid="schema-field-event_name-businessAttribute"]').within(() => + cy.contains("Add Attribute").click() + ); + cy.selectOptionInAttributeModal(businessAttribute); + cy.contains(businessAttribute); +}); + Cypress.Commands.add('selectOptionInTagTermModal', (text) => { cy.enterTextInTestId("tag-term-modal-input", text); cy.clickOptionWithTestId("tag-term-option"); @@ -294,6 +343,14 @@ Cypress.Commands.add('selectOptionInTagTermModal', (text) => { cy.get(selectorWithtestId(btn_id)).should("not.exist"); }); +Cypress.Commands.add('selectOptionInAttributeModal', (text) => { + cy.enterTextInTestId("business-attribute-modal-input", text); + cy.clickOptionWithTestId("business-attribute-option"); + let btn_id = "add-attribute-from-modal-btn"; + cy.clickOptionWithTestId(btn_id); + cy.get(selectorWithtestId(btn_id)).should("not.exist"); +}); + Cypress.Commands.add("removeDomainFromDataset", (urn, dataset_name, domain_urn) => { cy.goToDataset(urn, dataset_name); cy.get('.sidebar-domain-section [href="/domain/' + domain_urn + '"] .anticon-close').click(); diff --git a/smoke-test/tests/cypress/data.json b/smoke-test/tests/cypress/data.json index 0d56cfe9389182..391eba1fe93421 100644 --- a/smoke-test/tests/cypress/data.json +++ b/smoke-test/tests/cypress/data.json @@ -2082,5 +2082,57 @@ "contentType": "application/json" }, "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "businessAttributeInfo", + "aspect": { + "value": "{\n \"fieldPath\": \"CypressAttribute\",\n \"description\": \"CypressAttribute\",\n \"globalTags\": {\n \"tags\": [\n {\n \"tag\": \"urn:li:tag:Cypress\"\n }\n ]\n },\n \"glossaryTerms\": {\n \"terms\": [\n {\n \"urn\": \"urn:li:glossaryTerm:CypressNode.CypressTerm\"\n }\n ],\n \"auditStamp\": {\n \"time\": 1706889592683,\n \"actor\": \"urn:li:corpuser:datahub\"\n }\n },\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"CypressAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:37c81832-06e0-40b1-a682-858e1dd0d449", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "businessAttributeInfo", + "aspect": { + "value": "{\n \"fieldPath\": \"cypressTestAttribute\",\n \"description\": \"cypressTestAttribute\",\n \"customProperties\": {},\n \"created\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"lastModified\": {\n \"time\": 1706690081803,\n \"actor\": \"urn:li:corpuser:datahub\"\n },\n \"name\": \"cypressTestAttribute\",\n \"type\": {\n \"type\": {\n \"com.linkedin.schema.BooleanType\": {}\n }\n }\n }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "businessAttribute", + "entityUrn": "urn:li:businessAttribute:cypressTestAttribute", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\n \"owners\": [\n {\n \"owner\": \"urn:li:corpuser:datahub\",\n \"type\": \"TECHNICAL_OWNER\",\n \"typeUrn\": \"urn:li:ownershipType:__system__technical_owner\",\n \"source\": {\n \"type\": \"MANUAL\"\n }\n }\n ]\n }", + "contentType": "application/json" + }, + "systemMetadata": null } ] \ No newline at end of file diff --git a/test-models/src/main/pegasus/com/datahub/test/RefEntityAspect.pdl b/test-models/src/main/pegasus/com/datahub/test/RefEntityAspect.pdl new file mode 100644 index 00000000000000..2921cc2e389ab1 --- /dev/null +++ b/test-models/src/main/pegasus/com/datahub/test/RefEntityAspect.pdl @@ -0,0 +1,7 @@ +namespace com.datahub.test + + +/** + * A union of all supported metadata aspects for a RefEntity + */ +typeref RefEntityAspect = union[RefEntityKey, RefProperties] \ No newline at end of file diff --git a/test-models/src/main/pegasus/com/datahub/test/RefEntityAssociation.pdl b/test-models/src/main/pegasus/com/datahub/test/RefEntityAssociation.pdl new file mode 100644 index 00000000000000..9384a7d0d9a9c5 --- /dev/null +++ b/test-models/src/main/pegasus/com/datahub/test/RefEntityAssociation.pdl @@ -0,0 +1,8 @@ +namespace com.datahub.test + +import com.linkedin.common.Urn +import com.linkedin.common.Edge + +record RefEntityAssociation includes Edge{ + +} \ No newline at end of file diff --git a/test-models/src/main/pegasus/com/datahub/test/RefEntityKey.pdl b/test-models/src/main/pegasus/com/datahub/test/RefEntityKey.pdl new file mode 100644 index 00000000000000..2197ef81c4031f --- /dev/null +++ b/test-models/src/main/pegasus/com/datahub/test/RefEntityKey.pdl @@ -0,0 +1,17 @@ +namespace com.datahub.test + +import com.linkedin.common.Urn + +/** + * Key for Test Entity entity + */ +@Aspect = { + "name": "refEntityKey" +} +record RefEntityKey { + + /** + * A unique id + */ + id: string +} diff --git a/test-models/src/main/pegasus/com/datahub/test/RefEntityProperties.pdl b/test-models/src/main/pegasus/com/datahub/test/RefEntityProperties.pdl new file mode 100644 index 00000000000000..eab805e4fc7b52 --- /dev/null +++ b/test-models/src/main/pegasus/com/datahub/test/RefEntityProperties.pdl @@ -0,0 +1,31 @@ +namespace com.datahub.test + + +/** + * Additional properties associated with a RefEntity + */ +@Aspect = { + "name": "refEntityProperties" +} +record RefEntityProperties { + /** + * Display name of the RefEntity + */ + @Searchable = { + "fieldType": "WORD_GRAM", + "enableAutocomplete": true, + "boostScore": 10.0, + "fieldName": "displayName" + } + name: string + + /** + * Description of the RefEntity + */ + @Searchable = { + "fieldName": "editedFieldDescriptions", + "fieldType": "TEXT", + "boostScore": 0.1 + } + description: optional string +} \ No newline at end of file diff --git a/test-models/src/main/pegasus/com/datahub/test/RefProperties.pdl b/test-models/src/main/pegasus/com/datahub/test/RefProperties.pdl new file mode 100644 index 00000000000000..e04faeab3b0e7d --- /dev/null +++ b/test-models/src/main/pegasus/com/datahub/test/RefProperties.pdl @@ -0,0 +1,20 @@ +namespace com.datahub.test + +/** + * Properties associated with a Tag + */ +@Aspect = { + "name": "RefProperties" +} +record RefProperties { + /** + * Display name of the ref + */ + @Searchable = { + "fieldType": "WORD_GRAM", + "enableAutocomplete": true, + "boostScore": 10.0, + "fieldNameAliases": [ "_entityName" ] + } + name: string +} \ No newline at end of file diff --git a/test-models/src/main/pegasus/com/datahub/test/TestRefEntity.pdl b/test-models/src/main/pegasus/com/datahub/test/TestRefEntity.pdl new file mode 100644 index 00000000000000..b128f6780e4fbc --- /dev/null +++ b/test-models/src/main/pegasus/com/datahub/test/TestRefEntity.pdl @@ -0,0 +1,20 @@ +namespace com.datahub.test + +import com.linkedin.common.Urn + +@Entity = { + "name": "testRefEntity", + "keyAspect": "testRefEntityKey" +} +record TestRefEntity { + + /** + * Urn for the service + */ + urn: Urn + + /** + * The list of service aspects + */ + aspects: array[TestRefEntityAspect] +} \ No newline at end of file diff --git a/test-models/src/main/pegasus/com/datahub/test/TestRefEntityAspect.pdl b/test-models/src/main/pegasus/com/datahub/test/TestRefEntityAspect.pdl new file mode 100644 index 00000000000000..9c732c9678c6f2 --- /dev/null +++ b/test-models/src/main/pegasus/com/datahub/test/TestRefEntityAspect.pdl @@ -0,0 +1,6 @@ +namespace com.datahub.test + +/** + * A union of all supported metadata aspects for a RefEntity + */ +typeref TestRefEntityAspect = union[TestRefEntityKey, TestRefEntityInfo] \ No newline at end of file diff --git a/test-models/src/main/pegasus/com/datahub/test/TestRefEntityInfo.pdl b/test-models/src/main/pegasus/com/datahub/test/TestRefEntityInfo.pdl new file mode 100644 index 00000000000000..8116753a4b7274 --- /dev/null +++ b/test-models/src/main/pegasus/com/datahub/test/TestRefEntityInfo.pdl @@ -0,0 +1,49 @@ +namespace com.datahub.test + + +/** + * Additional properties associated with a RefEntity + */ +@Aspect = { + "name": "testRefEntityInfo" +} +record TestRefEntityInfo { + /** + * Display name of the testRefEntityInfo + */ + @Searchable = { + "fieldType": "WORD_GRAM", + "enableAutocomplete": true, + "boostScore": 10.0, + "fieldName": "displayName" + } + name: string + + /** + * Description of the RefEntity + */ + @Searchable = { + "fieldName": "editedFieldDescriptions", + "fieldType": "TEXT", + "boostScore": 0.1 + } + description: optional string + + +@SearchableRef = { + "/destinationUrn": { + "fieldName": "refEntityUrns", + "fieldType": "URN", + "refType" : "RefEntity" + } + } + refEntityAssociation: optional RefEntityAssociation + + @SearchableRef = { + "fieldName": "editedFieldDescriptionsRef", + "fieldType": "TEXT", + "boostScore": 0.5, + "refType" : "RefEntity" + } + refEntityAssociationText: optional string +} \ No newline at end of file diff --git a/test-models/src/main/pegasus/com/datahub/test/TestRefEntityKey.pdl b/test-models/src/main/pegasus/com/datahub/test/TestRefEntityKey.pdl new file mode 100644 index 00000000000000..0aab3d091d0ff9 --- /dev/null +++ b/test-models/src/main/pegasus/com/datahub/test/TestRefEntityKey.pdl @@ -0,0 +1,16 @@ +namespace com.datahub.test + + +/** + * Key for Test Ref Entity Defining parent entity with reference field + */ +@Aspect = { + "name": "testRefEntityKey" +} +record TestRefEntityKey { + + /** + * A unique id + */ + id: string +}