diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 6f2e250c17c34..b751849859be4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -59,6 +59,7 @@ import com.linkedin.datahub.graphql.generated.DataQualityContract; import com.linkedin.datahub.graphql.generated.Dataset; import com.linkedin.datahub.graphql.generated.DatasetStatsSummary; +import com.linkedin.datahub.graphql.generated.Deprecation; import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.ERModelRelationship; import com.linkedin.datahub.graphql.generated.ERModelRelationshipProperties; @@ -172,8 +173,11 @@ import com.linkedin.datahub.graphql.resolvers.form.BatchAssignFormResolver; import com.linkedin.datahub.graphql.resolvers.form.BatchRemoveFormResolver; import com.linkedin.datahub.graphql.resolvers.form.CreateDynamicFormAssignmentResolver; +import com.linkedin.datahub.graphql.resolvers.form.CreateFormResolver; +import com.linkedin.datahub.graphql.resolvers.form.DeleteFormResolver; import com.linkedin.datahub.graphql.resolvers.form.IsFormAssignedToMeResolver; import com.linkedin.datahub.graphql.resolvers.form.SubmitFormPromptResolver; +import com.linkedin.datahub.graphql.resolvers.form.UpdateFormResolver; import com.linkedin.datahub.graphql.resolvers.form.VerifyFormResolver; import com.linkedin.datahub.graphql.resolvers.glossary.AddRelatedTermsResolver; import com.linkedin.datahub.graphql.resolvers.glossary.CreateGlossaryNodeResolver; @@ -285,6 +289,9 @@ import com.linkedin.datahub.graphql.resolvers.settings.view.UpdateGlobalViewsSettingsResolver; import com.linkedin.datahub.graphql.resolvers.step.BatchGetStepStatesResolver; import com.linkedin.datahub.graphql.resolvers.step.BatchUpdateStepStatesResolver; +import com.linkedin.datahub.graphql.resolvers.structuredproperties.CreateStructuredPropertyResolver; +import com.linkedin.datahub.graphql.resolvers.structuredproperties.RemoveStructuredPropertiesResolver; +import com.linkedin.datahub.graphql.resolvers.structuredproperties.UpdateStructuredPropertyResolver; import com.linkedin.datahub.graphql.resolvers.structuredproperties.UpsertStructuredPropertiesResolver; import com.linkedin.datahub.graphql.resolvers.tag.CreateTagResolver; import com.linkedin.datahub.graphql.resolvers.tag.DeleteTagResolver; @@ -782,6 +789,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureBusinessAttributeResolver(builder); configureBusinessAttributeAssociationResolver(builder); configureConnectionResolvers(builder); + configureDeprecationResolvers(builder); } private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) { @@ -1316,10 +1324,23 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher( "upsertStructuredProperties", new UpsertStructuredPropertiesResolver(this.entityClient)) + .dataFetcher( + "removeStructuredProperties", + new RemoveStructuredPropertiesResolver(this.entityClient)) + .dataFetcher( + "createStructuredProperty", + new CreateStructuredPropertyResolver(this.entityClient)) + .dataFetcher( + "updateStructuredProperty", + new UpdateStructuredPropertyResolver(this.entityClient)) .dataFetcher("raiseIncident", new RaiseIncidentResolver(this.entityClient)) .dataFetcher( "updateIncidentStatus", - new UpdateIncidentStatusResolver(this.entityClient, this.entityService)); + new UpdateIncidentStatusResolver(this.entityClient, this.entityService)) + .dataFetcher( + "createForm", new CreateFormResolver(this.entityClient, this.formService)) + .dataFetcher("deleteForm", new DeleteFormResolver(this.entityClient)) + .dataFetcher("updateForm", new UpdateFormResolver(this.entityClient)); if (featureFlags.isBusinessAttributeEntityEnabled()) { typeWiring .dataFetcher( @@ -3143,4 +3164,14 @@ private void configureConnectionResolvers(final RuntimeWiring.Builder builder) { : null; }))); } + + private void configureDeprecationResolvers(final RuntimeWiring.Builder builder) { + builder.type( + "Deprecation", + typeWiring -> + typeWiring.dataFetcher( + "actorEntity", + new EntityTypeResolver( + entityTypes, (env) -> ((Deprecation) env.getSource()).getActorEntity()))); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java index c16e436a7805c..fa09a0fded5fb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java @@ -381,6 +381,20 @@ public static T restrictEntity(@Nonnull Object entity, Class clazz) { } } + public static boolean canManageStructuredProperties(@Nonnull QueryContext context) { + return AuthUtil.isAuthorized( + context.getAuthorizer(), + context.getActorUrn(), + PoliciesConfig.MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE); + } + + public static boolean canManageForms(@Nonnull QueryContext context) { + return AuthUtil.isAuthorized( + context.getAuthorizer(), + context.getActorUrn(), + PoliciesConfig.MANAGE_DOCUMENTATION_FORMS_PRIVILEGE); + } + public static boolean isAuthorized( @Nonnull Authorizer authorizer, @Nonnull String actor, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateFormResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateFormResolver.java new file mode 100644 index 0000000000000..e9962464059e6 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateFormResolver.java @@ -0,0 +1,83 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.CreateFormInput; +import com.linkedin.datahub.graphql.generated.CreatePromptInput; +import com.linkedin.datahub.graphql.generated.Form; +import com.linkedin.datahub.graphql.generated.FormPromptType; +import com.linkedin.datahub.graphql.resolvers.mutate.util.FormUtils; +import com.linkedin.datahub.graphql.types.form.FormMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.form.FormInfo; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.service.FormService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class CreateFormResolver implements DataFetcher> { + + private final EntityClient _entityClient; + private final FormService _formService; + + public CreateFormResolver( + @Nonnull final EntityClient entityClient, @Nonnull final FormService formService) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + _formService = Objects.requireNonNull(formService, "formService must not be null"); + } + + @Override + public CompletableFuture
get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + + final CreateFormInput input = + bindArgument(environment.getArgument("input"), CreateFormInput.class); + final FormInfo formInfo = FormUtils.mapFormInfo(input); + + return CompletableFuture.supplyAsync( + () -> { + try { + if (!AuthorizationUtils.canManageForms(context)) { + throw new AuthorizationException("Unable to create form. Please contact your admin."); + } + validatePrompts(input.getPrompts()); + + Urn formUrn = + _formService.createForm(context.getOperationContext(), formInfo, input.getId()); + EntityResponse response = + _entityClient.getV2( + context.getOperationContext(), Constants.FORM_ENTITY_NAME, formUrn, null); + return FormMapper.map(context, response); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } + + private void validatePrompts(@Nullable List prompts) { + if (prompts == null) { + return; + } + prompts.forEach( + prompt -> { + if (prompt.getType().equals(FormPromptType.STRUCTURED_PROPERTY) + || prompt.getType().equals(FormPromptType.FIELDS_STRUCTURED_PROPERTY)) { + if (prompt.getStructuredPropertyParams() == null) { + throw new IllegalArgumentException( + "Provided prompt with type STRUCTURED_PROPERTY or FIELDS_STRUCTURED_PROPERTY and no structured property params"); + } + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/DeleteFormResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/DeleteFormResolver.java new file mode 100644 index 0000000000000..eec6816042a40 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/DeleteFormResolver.java @@ -0,0 +1,65 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.DeleteFormInput; +import com.linkedin.entity.client.EntityClient; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DeleteFormResolver implements DataFetcher> { + + private final EntityClient _entityClient; + + public DeleteFormResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + + final DeleteFormInput input = + bindArgument(environment.getArgument("input"), DeleteFormInput.class); + final Urn formUrn = UrnUtils.getUrn(input.getUrn()); + + return CompletableFuture.supplyAsync( + () -> { + try { + if (!AuthorizationUtils.canManageForms(context)) { + throw new AuthorizationException("Unable to delete form. Please contact your admin."); + } + _entityClient.deleteEntity(context.getOperationContext(), formUrn); + // Asynchronously Delete all references to the entity (to return quickly) + CompletableFuture.runAsync( + () -> { + try { + _entityClient.deleteEntityReferences(context.getOperationContext(), formUrn); + } catch (Exception e) { + log.error( + String.format( + "Caught exception while attempting to clear all entity references for Form with urn %s", + formUrn), + e); + } + }); + + return true; + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/UpdateFormResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/UpdateFormResolver.java new file mode 100644 index 0000000000000..8b4d1debcd4db --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/UpdateFormResolver.java @@ -0,0 +1,98 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.Form; +import com.linkedin.datahub.graphql.generated.UpdateFormInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.FormUtils; +import com.linkedin.datahub.graphql.types.form.FormMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.form.FormType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.patch.builder.FormInfoPatchBuilder; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +public class UpdateFormResolver implements DataFetcher> { + + private final EntityClient _entityClient; + + public UpdateFormResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + + final UpdateFormInput input = + bindArgument(environment.getArgument("input"), UpdateFormInput.class); + final Urn formUrn = UrnUtils.getUrn(input.getUrn()); + + return CompletableFuture.supplyAsync( + () -> { + try { + if (!AuthorizationUtils.canManageForms(context)) { + throw new AuthorizationException("Unable to update form. Please contact your admin."); + } + if (!_entityClient.exists(context.getOperationContext(), formUrn)) { + throw new IllegalArgumentException( + String.format("Form with urn %s does not exist", formUrn)); + } + + FormInfoPatchBuilder patchBuilder = new FormInfoPatchBuilder().urn(formUrn); + if (input.getName() != null) { + patchBuilder.setName(input.getName()); + } + if (input.getDescription() != null) { + patchBuilder.setDescription(input.getDescription()); + } + if (input.getType() != null) { + patchBuilder.setType(FormType.valueOf(input.getType().toString())); + } + if (input.getPromptsToAdd() != null) { + patchBuilder.addPrompts(FormUtils.mapPromptsToAdd(input.getPromptsToAdd())); + } + if (input.getPromptsToRemove() != null) { + patchBuilder.removePrompts(input.getPromptsToRemove()); + } + if (input.getActors() != null) { + if (input.getActors().getOwners() != null) { + patchBuilder.setOwnershipForm(input.getActors().getOwners()); + } + if (input.getActors().getUsersToAdd() != null) { + input.getActors().getUsersToAdd().forEach(patchBuilder::addAssignedUser); + } + if (input.getActors().getUsersToRemove() != null) { + input.getActors().getUsersToRemove().forEach(patchBuilder::removeAssignedUser); + } + if (input.getActors().getGroupsToAdd() != null) { + input.getActors().getGroupsToAdd().forEach(patchBuilder::addAssignedGroup); + } + if (input.getActors().getGroupsToRemove() != null) { + input.getActors().getGroupsToRemove().forEach(patchBuilder::removeAssignedGroup); + } + } + _entityClient.ingestProposal( + context.getOperationContext(), patchBuilder.build(), false); + + EntityResponse response = + _entityClient.getV2( + context.getOperationContext(), Constants.FORM_ENTITY_NAME, formUrn, null); + return FormMapper.map(context, response); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java index 6caa858460c2f..17718f39c1238 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java @@ -1,11 +1,23 @@ package com.linkedin.datahub.graphql.resolvers.mutate.util; +import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.generated.CreateDynamicFormAssignmentInput; +import com.linkedin.datahub.graphql.generated.CreateFormInput; +import com.linkedin.datahub.graphql.generated.CreatePromptInput; +import com.linkedin.datahub.graphql.generated.FormActorAssignmentInput; +import com.linkedin.datahub.graphql.generated.StructuredPropertyParamsInput; import com.linkedin.datahub.graphql.generated.SubmitFormPromptInput; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.form.DynamicFormAssignment; +import com.linkedin.form.FormActorAssignment; import com.linkedin.form.FormInfo; +import com.linkedin.form.FormPrompt; +import com.linkedin.form.FormPromptArray; +import com.linkedin.form.FormPromptType; +import com.linkedin.form.FormType; +import com.linkedin.form.StructuredPropertyParams; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; @@ -14,7 +26,10 @@ import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.structured.PrimitivePropertyValueArray; +import java.util.List; import java.util.Objects; +import java.util.UUID; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -99,4 +114,103 @@ private static boolean isActorExplicitlyAssigned( || (formInfo.getActors().getGroups() != null && formInfo.getActors().getGroups().stream().anyMatch(group -> group.equals(actorUrn))); } + + @Nonnull + public static FormInfo mapFormInfo(@Nonnull final CreateFormInput input) { + Objects.requireNonNull(input, "input must not be null"); + + final FormInfo result = new FormInfo(); + result.setName(input.getName()); + if (input.getDescription() != null) { + result.setDescription(input.getDescription()); + } + if (input.getType() != null) { + result.setType(FormType.valueOf(input.getType().toString())); + } + if (input.getPrompts() != null) { + result.setPrompts(mapPrompts(input.getPrompts())); + } + if (input.getActors() != null) { + result.setActors(mapFormActorAssignment(input.getActors())); + } + + return result; + } + + @Nonnull + public static FormPromptArray mapPrompts(@Nonnull final List promptInputs) { + Objects.requireNonNull(promptInputs, "promptInputs must not be null"); + + final FormPromptArray result = new FormPromptArray(); + promptInputs.forEach( + promptInput -> { + result.add(mapPrompt(promptInput)); + }); + return result; + } + + @Nonnull + public static FormPrompt mapPrompt(@Nonnull final CreatePromptInput promptInput) { + Objects.requireNonNull(promptInput, "promptInput must not be null"); + + final FormPrompt result = new FormPrompt(); + String promptId = + promptInput.getId() != null ? promptInput.getId() : UUID.randomUUID().toString(); + result.setId(promptId); + result.setTitle(promptInput.getTitle()); + if (promptInput.getDescription() != null) { + result.setDescription(promptInput.getDescription()); + } + if (promptInput.getType() != null) { + result.setType(FormPromptType.valueOf(promptInput.getType().toString())); + } + if (promptInput.getStructuredPropertyParams() != null) { + result.setStructuredPropertyParams( + mapStructuredPropertyParams(promptInput.getStructuredPropertyParams())); + } + if (promptInput.getRequired() != null) { + result.setRequired(promptInput.getRequired()); + } + + return result; + } + + @Nonnull + public static StructuredPropertyParams mapStructuredPropertyParams( + @Nonnull final StructuredPropertyParamsInput paramsInput) { + Objects.requireNonNull(paramsInput, "paramsInput must not be null"); + + final StructuredPropertyParams result = new StructuredPropertyParams(); + result.setUrn(UrnUtils.getUrn(paramsInput.getUrn())); + return result; + } + + @Nonnull + public static FormActorAssignment mapFormActorAssignment( + @Nonnull final FormActorAssignmentInput input) { + Objects.requireNonNull(input, "input must not be null"); + + final FormActorAssignment result = new FormActorAssignment(); + if (input.getOwners() != null) { + result.setOwners(input.getOwners()); + } + if (input.getUsers() != null) { + UrnArray userUrns = new UrnArray(); + input.getUsers().forEach(user -> userUrns.add(UrnUtils.getUrn(user))); + result.setUsers(userUrns); + } + if (input.getGroups() != null) { + UrnArray groupUrns = new UrnArray(); + input.getGroups().forEach(group -> groupUrns.add(UrnUtils.getUrn(group))); + result.setUsers(groupUrns); + } + + return result; + } + + @Nonnull + public static List mapPromptsToAdd( + @Nonnull final List promptsToAdd) { + return promptsToAdd.stream().map(FormUtils::mapPrompt).collect(Collectors.toList()); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java new file mode 100644 index 0000000000000..3be7ea505abbf --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java @@ -0,0 +1,136 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME; + +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.StringArray; +import com.linkedin.data.template.StringArrayMap; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.CreateStructuredPropertyInput; +import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertyMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PropertyCardinality; +import com.linkedin.structured.PropertyValue; +import com.linkedin.structured.StructuredPropertyKey; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +public class CreateStructuredPropertyResolver + implements DataFetcher> { + + private final EntityClient _entityClient; + + public CreateStructuredPropertyResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + + final CreateStructuredPropertyInput input = + bindArgument(environment.getArgument("input"), CreateStructuredPropertyInput.class); + + return CompletableFuture.supplyAsync( + () -> { + try { + if (!AuthorizationUtils.canManageStructuredProperties(context)) { + throw new AuthorizationException( + "Unable to create structured property. Please contact your admin."); + } + final StructuredPropertyKey key = new StructuredPropertyKey(); + final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); + key.setId(id); + final Urn propertyUrn = + EntityKeyUtils.convertEntityKeyToUrn(key, STRUCTURED_PROPERTY_ENTITY_NAME); + StructuredPropertyDefinitionPatchBuilder builder = + new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn); + + builder.setQualifiedName(input.getQualifiedName()); + builder.setValueType(input.getValueType()); + input.getEntityTypes().forEach(builder::addEntityType); + if (input.getDisplayName() != null) { + builder.setDisplayName(input.getDisplayName()); + } + if (input.getDescription() != null) { + builder.setDescription(input.getDescription()); + } + if (input.getImmutable() != null) { + builder.setImmutable(input.getImmutable()); + } + if (input.getTypeQualifier() != null) { + buildTypeQualifier(input, builder); + } + if (input.getAllowedValues() != null) { + buildAllowedValues(input, builder); + } + if (input.getCardinality() != null) { + builder.setCardinality( + PropertyCardinality.valueOf(input.getCardinality().toString())); + } + + MetadataChangeProposal mcp = builder.build(); + _entityClient.ingestProposal(context.getOperationContext(), mcp, false); + + EntityResponse response = + _entityClient.getV2( + context.getOperationContext(), + STRUCTURED_PROPERTY_ENTITY_NAME, + propertyUrn, + null); + return StructuredPropertyMapper.map(context, response); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } + + private void buildTypeQualifier( + @Nonnull final CreateStructuredPropertyInput input, + @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) { + if (input.getTypeQualifier().getAllowedTypes() != null) { + final StringArrayMap typeQualifier = new StringArrayMap(); + StringArray allowedTypes = new StringArray(); + allowedTypes.addAll(input.getTypeQualifier().getAllowedTypes()); + typeQualifier.put("allowedTypes", allowedTypes); + builder.setTypeQualifier(typeQualifier); + } + } + + private void buildAllowedValues( + @Nonnull final CreateStructuredPropertyInput input, + @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) { + input + .getAllowedValues() + .forEach( + allowedValueInput -> { + PropertyValue value = new PropertyValue(); + PrimitivePropertyValue primitiveValue = new PrimitivePropertyValue(); + if (allowedValueInput.getStringValue() != null) { + primitiveValue.setString(allowedValueInput.getStringValue()); + } + if (allowedValueInput.getNumberValue() != null) { + primitiveValue.setDouble(allowedValueInput.getNumberValue().doubleValue()); + } + value.setValue(primitiveValue); + value.setDescription(allowedValueInput.getDescription(), SetMode.IGNORE_NULL); + builder.addAllowedValue(value); + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java new file mode 100644 index 0000000000000..ea8c6dac36a4a --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java @@ -0,0 +1,103 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.RemoveStructuredPropertiesInput; +import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertiesMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertiesPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.structured.StructuredProperties; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +public class RemoveStructuredPropertiesResolver + implements DataFetcher< + CompletableFuture> { + + private final EntityClient _entityClient; + + public RemoveStructuredPropertiesResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get( + final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + + final RemoveStructuredPropertiesInput input = + bindArgument(environment.getArgument("input"), RemoveStructuredPropertiesInput.class); + final Urn assetUrn = UrnUtils.getUrn(input.getAssetUrn()); + + return CompletableFuture.supplyAsync( + () -> { + try { + // check authorization first + if (!AuthorizationUtils.canEditProperties(assetUrn, context)) { + throw new AuthorizationException( + String.format( + "Not authorized to update properties on the gives urn %s", assetUrn)); + } + + if (!_entityClient.exists(context.getOperationContext(), assetUrn)) { + throw new RuntimeException( + String.format("Asset with provided urn %s does not exist", assetUrn)); + } + + StructuredPropertiesPatchBuilder patchBuilder = + new StructuredPropertiesPatchBuilder().urn(assetUrn); + + input + .getStructuredPropertyUrns() + .forEach( + propertyUrn -> { + patchBuilder.removeProperty(UrnUtils.getUrn(propertyUrn)); + }); + + // ingest change proposal + final MetadataChangeProposal structuredPropertiesProposal = patchBuilder.build(); + + _entityClient.ingestProposal( + context.getOperationContext(), structuredPropertiesProposal, false); + + EntityResponse response = + _entityClient.getV2( + context.getOperationContext(), + assetUrn.getEntityType(), + assetUrn, + ImmutableSet.of(Constants.STRUCTURED_PROPERTIES_ASPECT_NAME)); + + if (response == null + || response.getAspects().get(Constants.STRUCTURED_PROPERTIES_ASPECT_NAME) == null) { + throw new RuntimeException( + String.format("Failed to fetch structured properties from entity %s", assetUrn)); + } + + StructuredProperties structuredProperties = + new StructuredProperties( + response + .getAspects() + .get(Constants.STRUCTURED_PROPERTIES_ASPECT_NAME) + .getValue() + .data()); + + return StructuredPropertiesMapper.map(context, structuredProperties); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java new file mode 100644 index 0000000000000..2549f303bacd9 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolver.java @@ -0,0 +1,129 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.StringArray; +import com.linkedin.data.template.StringArrayMap; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.UpdateStructuredPropertyInput; +import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertyMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PropertyCardinality; +import com.linkedin.structured.PropertyValue; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +public class UpdateStructuredPropertyResolver + implements DataFetcher> { + + private final EntityClient _entityClient; + + public UpdateStructuredPropertyResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + + final UpdateStructuredPropertyInput input = + bindArgument(environment.getArgument("input"), UpdateStructuredPropertyInput.class); + + return CompletableFuture.supplyAsync( + () -> { + try { + if (!AuthorizationUtils.canManageStructuredProperties(context)) { + throw new AuthorizationException( + "Unable to update structured property. Please contact your admin."); + } + final Urn propertyUrn = UrnUtils.getUrn(input.getUrn()); + StructuredPropertyDefinitionPatchBuilder builder = + new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn); + + if (input.getDisplayName() != null) { + builder.setDisplayName(input.getDisplayName()); + } + if (input.getDescription() != null) { + builder.setDescription(input.getDescription()); + } + if (input.getImmutable() != null) { + builder.setImmutable(input.getImmutable()); + } + if (input.getTypeQualifier() != null) { + buildTypeQualifier(input, builder); + } + if (input.getNewAllowedValues() != null) { + buildAllowedValues(input, builder); + } + if (input.getSetCardinalityAsMultiple() != null) { + builder.setCardinality(PropertyCardinality.MULTIPLE); + } + if (input.getNewEntityTypes() != null) { + input.getNewEntityTypes().forEach(builder::addEntityType); + } + + MetadataChangeProposal mcp = builder.build(); + _entityClient.ingestProposal(context.getOperationContext(), mcp, false); + + EntityResponse response = + _entityClient.getV2( + context.getOperationContext(), + STRUCTURED_PROPERTY_ENTITY_NAME, + propertyUrn, + null); + return StructuredPropertyMapper.map(context, response); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } + + private void buildTypeQualifier( + @Nonnull final UpdateStructuredPropertyInput input, + @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) { + if (input.getTypeQualifier().getNewAllowedTypes() != null) { + final StringArrayMap typeQualifier = new StringArrayMap(); + StringArray allowedTypes = new StringArray(); + allowedTypes.addAll(input.getTypeQualifier().getNewAllowedTypes()); + typeQualifier.put("allowedTypes", allowedTypes); + builder.setTypeQualifier(typeQualifier); + } + } + + private void buildAllowedValues( + @Nonnull final UpdateStructuredPropertyInput input, + @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) { + input + .getNewAllowedValues() + .forEach( + allowedValueInput -> { + PropertyValue value = new PropertyValue(); + PrimitivePropertyValue primitiveValue = new PrimitivePropertyValue(); + if (allowedValueInput.getStringValue() != null) { + primitiveValue.setString(allowedValueInput.getStringValue()); + } + if (allowedValueInput.getNumberValue() != null) { + primitiveValue.setDouble(allowedValueInput.getNumberValue().doubleValue()); + } + value.setValue(primitiveValue); + value.setDescription(allowedValueInput.getDescription(), SetMode.IGNORE_NULL); + builder.addAllowedValue(value); + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DeprecationMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DeprecationMapper.java index 8c3d72edfed25..6959a6dcbd039 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DeprecationMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DeprecationMapper.java @@ -20,6 +20,7 @@ public Deprecation apply( @Nullable QueryContext context, @Nonnull final com.linkedin.common.Deprecation input) { final Deprecation result = new Deprecation(); result.setActor(input.getActor().toString()); + result.setActorEntity(UrnToEntityMapper.map(context, input.getActor())); result.setDeprecated(input.isDeprecated()); result.setDecommissionTime(input.getDecommissionTime()); result.setNote(input.getNote()); diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 89c7b4a4cd055..246ace2fc0f5f 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -8183,6 +8183,11 @@ type Deprecation { The user who will be credited for modifying this deprecation content """ actor: String + + """ + The hydrated user who will be credited for modifying this deprecation content + """ + actorEntity: Entity } """ diff --git a/datahub-graphql-core/src/main/resources/forms.graphql b/datahub-graphql-core/src/main/resources/forms.graphql index f5e5fa74e3dc9..4a4e270509596 100644 --- a/datahub-graphql-core/src/main/resources/forms.graphql +++ b/datahub-graphql-core/src/main/resources/forms.graphql @@ -3,6 +3,21 @@ extend type Mutation { Remove a form from a given list of entities. """ batchRemoveForm(input: BatchRemoveFormInput!): Boolean! + + """ + Create a new form based on the input + """ + createForm(input: CreateFormInput!): Form! + + """ + Delete a given form + """ + deleteForm(input: DeleteFormInput!): Boolean! + + """ + Update an existing form based on the input + """ + updateForm(input: UpdateFormInput!): Form! } """ @@ -398,3 +413,184 @@ input BatchRemoveFormInput { """ entityUrns: [String!]! } + +""" +Input for batch removing a form from different entities +""" +input CreateFormInput { + """ + Advanced: Optionally provide an ID to create a form urn from + """ + id: String + + """ + The name of the form being created + """ + name: String! + + """ + The optional description of the form being created + """ + description: String + + """ + The type of this form, whether it's verification or completion. Default is completion. + """ + type: FormType + + """ + The type of this form, whether it's verification or completion. Default is completion. + """ + prompts: [CreatePromptInput!] + + """ + Information on how this form should be assigned to users/groups + """ + actors: FormActorAssignmentInput +} + +""" +Input for creating form prompts +""" +input CreatePromptInput { + """ + Advanced: Optionally provide an ID to this prompt. All prompt IDs must be globally unique. + """ + id: String + + """ + The title of the prompt + """ + title: String! + + """ + The optional description of the prompt + """ + description: String + + """ + The type of the prompt. + """ + type: FormPromptType! + + """ + The params required if this prompt type is STRUCTURED_PROPERTY or FIELDS_STRUCTURED_PROPERTY + """ + structuredPropertyParams: StructuredPropertyParamsInput + + """ + Whether this prompt will be required or not. Default is false. + """ + required: Boolean + +} + +""" +Input for assigning a form to actors +""" +input FormActorAssignmentInput { + """ + Whether this form will be applied to owners of associated entities or not. Default is true. + """ + owners: Boolean + + """ + The optional list of user urns to assign this form to + """ + users: [String!] + + """ + The optional list of group urns to assign this form to + """ + groups: [String!] +} + +""" +Input for a structured property type prompt +""" +input StructuredPropertyParamsInput { + """ + The urn of the structured property for a given form prompt + """ + urn: String! +} + +""" +Input for updating a form +""" +input UpdateFormInput { + """ + The urn of the form being updated + """ + urn: String! + + """ + The new name of the form + """ + name: String + + """ + The new description of the form + """ + description: String + + """ + The new type of the form + """ + type: FormType + + """ + The new prompts being added to this form + """ + promptsToAdd: [CreatePromptInput!] + + """ + The IDs of the prompts to remove from this form + """ + promptsToRemove: [String!] + + """ + Information on how this form should be assigned to users/groups + """ + actors: FormActorAssignmentUpdateInput +} + +""" +Update input for assigning a form to actors +""" +input FormActorAssignmentUpdateInput { + """ + Whether this form will be applied to owners of associated entities or not. Default is true. + """ + owners: Boolean + + """ + The optional list of user urns to assign this form to + """ + usersToAdd: [String!] + + """ + The users being removed from being assigned to this form + """ + usersToRemove: [String!] + + """ + The optional list of group urns to assign this form to + """ + groupsToAdd: [String!] + + """ + The groups being removed from being assigned to this form + """ + groupsToRemove: [String!] +} + +""" +Input for deleting a form +""" +input DeleteFormInput { + """ + The urn of the form that is being deleted + """ + urn: String! +} diff --git a/datahub-graphql-core/src/main/resources/properties.graphql b/datahub-graphql-core/src/main/resources/properties.graphql index 120154e930d59..dfe8468645681 100644 --- a/datahub-graphql-core/src/main/resources/properties.graphql +++ b/datahub-graphql-core/src/main/resources/properties.graphql @@ -3,6 +3,21 @@ extend type Mutation { Upsert structured properties onto a given asset """ upsertStructuredProperties(input: UpsertStructuredPropertiesInput!): StructuredProperties! + + """ + Upsert structured properties onto a given asset + """ + removeStructuredProperties(input: RemoveStructuredPropertiesInput!): StructuredProperties! + + """ + Create a new structured property + """ + createStructuredProperty(input: CreateStructuredPropertyInput!): StructuredPropertyEntity! + + """ + Update an existing structured property + """ + updateStructuredProperty(input: UpdateStructuredPropertyInput!): StructuredPropertyEntity! } """ @@ -184,6 +199,21 @@ input UpsertStructuredPropertiesInput { structuredPropertyInputParams: [StructuredPropertyInputParams!]! } +""" +Input for removing structured properties on a given asset +""" +input RemoveStructuredPropertiesInput { + """ + The urn of the asset that we are removing properties from + """ + assetUrn: String! + + """ + The list of structured properties you want to remove from this asset + """ + structuredPropertyUrns: [String!]! +} + """ A data type registered in DataHub """ @@ -268,3 +298,152 @@ type DataTypeInfo { """ description: String } + +""" +Input for creating a new structured property entity +""" +input CreateStructuredPropertyInput { + """ + (Advanced) An optional unique ID to use when creating the urn of this entity + """ + id: String + + """ + The unique fully qualified name of this structured property, dot delimited. + """ + qualifiedName: String! + + """ + The optional display name for this property + """ + displayName: String + + """ + The optional description for this property + """ + description: String + + """ + Whether the property will be mutable once it is applied or not. Default is false. + """ + immutable: Boolean + + """ + The urn of the value type that this structured property accepts. + For example: urn:li:dataType:datahub.string or urn:li:dataType:datahub.date + """ + valueType: String! + + """ + The optional input for specifying specific entity types as values + """ + typeQualifier: TypeQualifierInput + + """ + The optional input for specifying a list of allowed values + """ + allowedValues: [AllowedValueInput!] + + """ + The optional input for specifying if one or multiple values can be applied. + Default is one value (single cardinality) + """ + cardinality: PropertyCardinality + + """ + The list of entity types that this property can be applied to. + For example: ["urn:li:entityType:datahub.dataset"] + """ + entityTypes: [String!]! +} + +""" +Input for specifying specific entity types as values +""" +input TypeQualifierInput { + """ + The list of allowed entity types as urns (ie. ["urn:li:entityType:datahub.corpuser"]) + """ + allowedTypes: [String!] +} + +""" +An input entry for an allowed value for a structured property +""" +input AllowedValueInput { + """ + The allowed string value if the value is of type string + Either this or numberValue is required. + """ + stringValue: String + + """ + The allowed number value if the value is of type number. + Either this or stringValue is required. + """ + numberValue: Float + + """ + The description of this allowed value + """ + description: String +} + +""" +Input for updating an existing structured property entity +""" +input UpdateStructuredPropertyInput { + """ + The urn of the structured property being updated + """ + urn: String! + + """ + The optional display name for this property + """ + displayName: String + + """ + The optional description for this property + """ + description: String + + """ + Whether the property will be mutable once it is applied or not. Default is false. + """ + immutable: Boolean + + """ + The optional input for specifying specific entity types as values + """ + typeQualifier: UpdateTypeQualifierInput + + """ + Append to the list of allowed values for this property. + For backwards compatibility, this is append only. + """ + newAllowedValues: [AllowedValueInput!] + + """ + Set to true if you want to change the cardinality of this structured property + to multiple. Cannot change from multiple to single for backwards compatibility reasons. + """ + setCardinalityAsMultiple: Boolean + + """ + Append to the list of entity types that this property can be applied to. + For backwards compatibility, this is append only. + """ + newEntityTypes: [String!] +} + +""" +Input for updating specifying specific entity types as values +""" +input UpdateTypeQualifierInput { + """ + Append to the list of allowed entity types as urns for this property (ie. ["urn:li:entityType:datahub.corpuser"]) + For backwards compatibility, this is append only. + """ + newAllowedTypes: [String!] +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/CreateFormResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/CreateFormResolverTest.java new file mode 100644 index 0000000000000..65f51830ee148 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/CreateFormResolverTest.java @@ -0,0 +1,116 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.*; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.CreateFormInput; +import com.linkedin.datahub.graphql.generated.Form; +import com.linkedin.datahub.graphql.generated.FormType; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.form.FormInfo; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.service.FormService; +import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class CreateFormResolverTest { + private static final String TEST_FORM_URN = "urn:li:form:1"; + + private static final CreateFormInput TEST_INPUT = + new CreateFormInput(null, "test name", null, FormType.VERIFICATION, new ArrayList<>(), null); + + @Test + public void testGetSuccess() throws Exception { + FormService mockFormService = initMockFormService(true); + EntityClient mockEntityClient = initMockEntityClient(); + CreateFormResolver resolver = new CreateFormResolver(mockEntityClient, mockFormService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Form form = resolver.get(mockEnv).get(); + + assertEquals(form.getUrn(), TEST_FORM_URN); + + // Validate that we called create on the service + Mockito.verify(mockFormService, Mockito.times(1)) + .createForm(any(), any(FormInfo.class), Mockito.eq(null)); + } + + @Test + public void testGetUnauthorized() throws Exception { + FormService mockFormService = initMockFormService(true); + EntityClient mockEntityClient = initMockEntityClient(); + CreateFormResolver resolver = new CreateFormResolver(mockEntityClient, mockFormService); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call create on the service + Mockito.verify(mockFormService, Mockito.times(0)) + .createForm(any(), any(FormInfo.class), Mockito.eq(null)); + } + + @Test + public void testGetFailure() throws Exception { + FormService mockFormService = initMockFormService(false); + EntityClient mockEntityClient = initMockEntityClient(); + CreateFormResolver resolver = new CreateFormResolver(mockEntityClient, mockFormService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we called create on the service + Mockito.verify(mockFormService, Mockito.times(1)) + .createForm(any(), any(FormInfo.class), Mockito.eq(null)); + } + + private FormService initMockFormService(final boolean shouldSucceed) throws Exception { + FormService service = Mockito.mock(FormService.class); + + if (shouldSucceed) { + Mockito.when(service.createForm(any(), Mockito.any(), Mockito.any())) + .thenReturn(UrnUtils.getUrn("urn:li:form:1")); + } else { + Mockito.when(service.createForm(any(), Mockito.any(), Mockito.any())) + .thenThrow(new RuntimeException()); + } + + return service; + } + + private EntityClient initMockEntityClient() throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + EntityResponse response = new EntityResponse(); + response.setEntityName(Constants.FORM_ENTITY_NAME); + response.setUrn(UrnUtils.getUrn(TEST_FORM_URN)); + response.setAspects(new EnvelopedAspectMap()); + Mockito.when( + client.getV2(any(), Mockito.eq(Constants.FORM_ENTITY_NAME), any(), Mockito.eq(null))) + .thenReturn(response); + + return client; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/DeleteFormResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/DeleteFormResolverTest.java new file mode 100644 index 0000000000000..ded79ed9a0018 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/DeleteFormResolverTest.java @@ -0,0 +1,90 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DeleteFormInput; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class DeleteFormResolverTest { + private static final String TEST_FORM_URN = "urn:li:form:1"; + + private static final DeleteFormInput TEST_INPUT = new DeleteFormInput(TEST_FORM_URN); + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + DeleteFormResolver resolver = new DeleteFormResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Boolean success = resolver.get(mockEnv).get(); + assertTrue(success); + + // Validate that we called delete + Mockito.verify(mockEntityClient, Mockito.times(1)) + .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN))); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + DeleteFormResolver resolver = new DeleteFormResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call delete and delete references + Mockito.verify(mockEntityClient, Mockito.times(0)) + .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN))); + Mockito.verify(mockEntityClient, Mockito.times(0)) + .deleteEntityReferences(any(), Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN))); + } + + @Test + public void testGetFailure() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(false); + DeleteFormResolver resolver = new DeleteFormResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that deleteEntity was called, but since it failed, delete references was not called + Mockito.verify(mockEntityClient, Mockito.times(1)) + .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN))); + Mockito.verify(mockEntityClient, Mockito.times(0)) + .deleteEntityReferences(any(), Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN))); + } + + private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + if (!shouldSucceed) { + Mockito.doThrow(new RemoteInvocationException()).when(client).deleteEntity(any(), any()); + } + return client; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/UpdateFormResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/UpdateFormResolverTest.java new file mode 100644 index 0000000000000..6a4b99742f7fd --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/UpdateFormResolverTest.java @@ -0,0 +1,105 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.*; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Form; +import com.linkedin.datahub.graphql.generated.UpdateFormInput; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class UpdateFormResolverTest { + private static final String TEST_FORM_URN = "urn:li:form:1"; + + private static final UpdateFormInput TEST_INPUT = + new UpdateFormInput(TEST_FORM_URN, "new name", null, null, null, null, null); + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + UpdateFormResolver resolver = new UpdateFormResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Form form = resolver.get(mockEnv).get(); + + assertEquals(form.getUrn(), TEST_FORM_URN); + + // Validate that we called ingest + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + UpdateFormResolver resolver = new UpdateFormResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call ingest + Mockito.verify(mockEntityClient, Mockito.times(0)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetFailure() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(false); + UpdateFormResolver resolver = new UpdateFormResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that ingest was called, but that caused a failure + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + EntityResponse response = new EntityResponse(); + response.setEntityName(Constants.FORM_ENTITY_NAME); + response.setUrn(UrnUtils.getUrn(TEST_FORM_URN)); + response.setAspects(new EnvelopedAspectMap()); + if (shouldSucceed) { + Mockito.when( + client.getV2(any(), Mockito.eq(Constants.FORM_ENTITY_NAME), any(), Mockito.eq(null))) + .thenReturn(response); + } else { + Mockito.when( + client.getV2(any(), Mockito.eq(Constants.FORM_ENTITY_NAME), any(), Mockito.eq(null))) + .thenThrow(new RemoteInvocationException()); + } + + Mockito.when(client.exists(any(), Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN)))).thenReturn(true); + + return client; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java new file mode 100644 index 0000000000000..72cdb78542e41 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java @@ -0,0 +1,126 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.CreateStructuredPropertyInput; +import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class CreateStructuredPropertyResolverTest { + private static final String TEST_STRUCTURED_PROPERTY_URN = "urn:li:structuredProperty:1"; + + private static final CreateStructuredPropertyInput TEST_INPUT = + new CreateStructuredPropertyInput( + null, + "io.acryl.test", + "Display Name", + "description", + true, + null, + null, + null, + null, + new ArrayList<>()); + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + CreateStructuredPropertyResolver resolver = + new CreateStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + StructuredPropertyEntity prop = resolver.get(mockEnv).get(); + + assertEquals(prop.getUrn(), TEST_STRUCTURED_PROPERTY_URN); + + // Validate that we called ingest + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + CreateStructuredPropertyResolver resolver = + new CreateStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call ingest + Mockito.verify(mockEntityClient, Mockito.times(0)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetFailure() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(false); + CreateStructuredPropertyResolver resolver = + new CreateStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that ingest was called, but that caused a failure + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + EntityResponse response = new EntityResponse(); + response.setEntityName(Constants.STRUCTURED_PROPERTY_ENTITY_NAME); + response.setUrn(UrnUtils.getUrn(TEST_STRUCTURED_PROPERTY_URN)); + response.setAspects(new EnvelopedAspectMap()); + if (shouldSucceed) { + Mockito.when( + client.getV2( + any(), + Mockito.eq(Constants.STRUCTURED_PROPERTY_ENTITY_NAME), + any(), + Mockito.eq(null))) + .thenReturn(response); + } else { + Mockito.when( + client.getV2( + any(), + Mockito.eq(Constants.STRUCTURED_PROPERTY_ENTITY_NAME), + any(), + Mockito.eq(null))) + .thenThrow(new RemoteInvocationException()); + } + + return client; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolverTest.java new file mode 100644 index 0000000000000..f7882bb2c93a8 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolverTest.java @@ -0,0 +1,123 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.assertThrows; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.RemoveStructuredPropertiesInput; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.structured.StructuredProperties; +import com.linkedin.structured.StructuredPropertyValueAssignmentArray; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class RemoveStructuredPropertiesResolverTest { + + private static final String TEST_DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD)"; + private static final String PROPERTY_URN_1 = "urn:li:structuredProperty:test1"; + private static final String PROPERTY_URN_2 = "urn:li:structuredProperty:test2"; + + private static final RemoveStructuredPropertiesInput TEST_INPUT = + new RemoveStructuredPropertiesInput( + TEST_DATASET_URN, ImmutableList.of(PROPERTY_URN_1, PROPERTY_URN_2)); + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(); + RemoveStructuredPropertiesResolver resolver = + new RemoveStructuredPropertiesResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + resolver.get(mockEnv).get(); + + // Validate that we called ingest + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(); + RemoveStructuredPropertiesResolver resolver = + new RemoveStructuredPropertiesResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call ingest + Mockito.verify(mockEntityClient, Mockito.times(0)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetThrowsError() throws Exception { + // if the entity you are trying to remove properties from doesn't exist + EntityClient mockEntityClient = Mockito.mock(EntityClient.class); + Mockito.when(mockEntityClient.exists(any(), Mockito.eq(UrnUtils.getUrn(TEST_DATASET_URN)))) + .thenReturn(false); + RemoveStructuredPropertiesResolver resolver = + new RemoveStructuredPropertiesResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call ingest + Mockito.verify(mockEntityClient, Mockito.times(0)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + private EntityClient initMockEntityClient() throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + EntityResponse response = new EntityResponse(); + response.setEntityName(Constants.DATASET_ENTITY_NAME); + response.setUrn(UrnUtils.getUrn(TEST_DATASET_URN)); + final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); + StructuredProperties properties = new StructuredProperties(); + properties.setProperties(new StructuredPropertyValueAssignmentArray()); + aspectMap.put( + STRUCTURED_PROPERTIES_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(properties.data()))); + response.setAspects(aspectMap); + Mockito.when( + client.getV2( + any(), + Mockito.eq(Constants.DATASET_ENTITY_NAME), + Mockito.eq(UrnUtils.getUrn(TEST_DATASET_URN)), + Mockito.eq(ImmutableSet.of(Constants.STRUCTURED_PROPERTIES_ASPECT_NAME)))) + .thenReturn(response); + Mockito.when(client.exists(any(), Mockito.eq(UrnUtils.getUrn(TEST_DATASET_URN)))) + .thenReturn(true); + + return client; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java new file mode 100644 index 0000000000000..971a53de9473b --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java @@ -0,0 +1,123 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.UpdateStructuredPropertyInput; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class UpdateStructuredPropertyResolverTest { + private static final String TEST_STRUCTURED_PROPERTY_URN = "urn:li:structuredProperty:1"; + + private static final UpdateStructuredPropertyInput TEST_INPUT = + new UpdateStructuredPropertyInput( + TEST_STRUCTURED_PROPERTY_URN, + "New Display Name", + "new description", + true, + null, + null, + null, + null); + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + UpdateStructuredPropertyResolver resolver = + new UpdateStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + StructuredPropertyEntity prop = resolver.get(mockEnv).get(); + + assertEquals(prop.getUrn(), TEST_STRUCTURED_PROPERTY_URN); + + // Validate that we called ingest + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + UpdateStructuredPropertyResolver resolver = + new UpdateStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call ingest + Mockito.verify(mockEntityClient, Mockito.times(0)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + @Test + public void testGetFailure() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(false); + UpdateStructuredPropertyResolver resolver = + new UpdateStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that ingest was called, but that caused a failure + Mockito.verify(mockEntityClient, Mockito.times(1)) + .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + } + + private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + EntityResponse response = new EntityResponse(); + response.setEntityName(Constants.STRUCTURED_PROPERTY_ENTITY_NAME); + response.setUrn(UrnUtils.getUrn(TEST_STRUCTURED_PROPERTY_URN)); + response.setAspects(new EnvelopedAspectMap()); + if (shouldSucceed) { + Mockito.when( + client.getV2( + any(), + Mockito.eq(Constants.STRUCTURED_PROPERTY_ENTITY_NAME), + any(), + Mockito.eq(null))) + .thenReturn(response); + } else { + Mockito.when( + client.getV2( + any(), + Mockito.eq(Constants.STRUCTURED_PROPERTY_ENTITY_NAME), + any(), + Mockito.eq(null))) + .thenThrow(new RemoteInvocationException()); + } + + return client; + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/FormInfoPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/FormInfoPatchBuilder.java new file mode 100644 index 0000000000000..13d9c07b04aee --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/FormInfoPatchBuilder.java @@ -0,0 +1,154 @@ +package com.linkedin.metadata.aspect.patch.builder; + +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.*; + +import com.datahub.util.RecordUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.linkedin.form.FormPrompt; +import com.linkedin.form.FormType; +import com.linkedin.metadata.aspect.patch.PatchOperationType; +import java.util.List; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.commons.lang3.tuple.ImmutableTriple; + +public class FormInfoPatchBuilder extends AbstractMultiFieldPatchBuilder { + + public static final String PATH_DELIM = "/"; + public static final String NAME_FIELD = "name"; + public static final String DESCRIPTION_FIELD = "description"; + public static final String TYPE_FIELD = "type"; + public static final String PROMPTS_FIELD = "prompts"; + public static final String ACTORS_FIELD = "actors"; + public static final String OWNERS_FIELD = "owners"; + public static final String USERS_FIELD = "users"; + public static final String GROUPS_FIELD = "groups"; + + public FormInfoPatchBuilder setName(@Nonnull String name) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), PATH_DELIM + NAME_FIELD, instance.textNode(name))); + return this; + } + + public FormInfoPatchBuilder setDescription(@Nullable String description) { + if (description == null) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.REMOVE.getValue(), PATH_DELIM + DESCRIPTION_FIELD, null)); + } else { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + DESCRIPTION_FIELD, + instance.textNode(description))); + } + return this; + } + + public FormInfoPatchBuilder setType(@Nonnull FormType formType) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + TYPE_FIELD, + instance.textNode(formType.toString()))); + return this; + } + + public FormInfoPatchBuilder addPrompt(@Nonnull FormPrompt prompt) { + try { + ObjectNode promptNode = + (ObjectNode) new ObjectMapper().readTree(RecordUtils.toJsonString(prompt)); + pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + PROMPTS_FIELD + PATH_DELIM + prompt.getId(), + promptNode)); + return this; + } catch (JsonProcessingException e) { + throw new IllegalArgumentException( + "Failed to add prompt, failed to parse provided aspect json.", e); + } + } + + public FormInfoPatchBuilder addPrompts(@Nonnull List prompts) { + try { + prompts.forEach(this::addPrompt); + return this; + } catch (Exception e) { + throw new IllegalArgumentException("Failed to add prompts.", e); + } + } + + public FormInfoPatchBuilder removePrompt(@Nonnull String promptId) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.REMOVE.getValue(), + PATH_DELIM + PROMPTS_FIELD + PATH_DELIM + promptId, + null)); + return this; + } + + public FormInfoPatchBuilder removePrompts(@Nonnull List promptIds) { + promptIds.forEach(this::removePrompt); + return this; + } + + public FormInfoPatchBuilder setOwnershipForm(boolean isOwnershipForm) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + ACTORS_FIELD + PATH_DELIM + OWNERS_FIELD, + instance.booleanNode(isOwnershipForm))); + return this; + } + + public FormInfoPatchBuilder addAssignedUser(@Nonnull String userUrn) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + ACTORS_FIELD + PATH_DELIM + USERS_FIELD + PATH_DELIM + userUrn, + instance.textNode(userUrn))); + return this; + } + + public FormInfoPatchBuilder removeAssignedUser(@Nonnull String userUrn) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.REMOVE.getValue(), + PATH_DELIM + ACTORS_FIELD + PATH_DELIM + USERS_FIELD + PATH_DELIM + userUrn, + instance.textNode(userUrn))); + return this; + } + + public FormInfoPatchBuilder addAssignedGroup(@Nonnull String groupUrn) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + ACTORS_FIELD + PATH_DELIM + GROUPS_FIELD + PATH_DELIM + groupUrn, + instance.textNode(groupUrn))); + return this; + } + + public FormInfoPatchBuilder removeAssignedGroup(@Nonnull String groupUrn) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.REMOVE.getValue(), + PATH_DELIM + ACTORS_FIELD + PATH_DELIM + GROUPS_FIELD + PATH_DELIM + groupUrn, + instance.textNode(groupUrn))); + return this; + } + + @Override + protected String getAspectName() { + return FORM_INFO_ASPECT_NAME; + } + + @Override + protected String getEntityType() { + return FORM_ENTITY_NAME; + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertiesPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertiesPatchBuilder.java index fab81e0af5bf5..b568df5054aae 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertiesPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertiesPatchBuilder.java @@ -3,12 +3,11 @@ import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.ValueNode; import com.linkedin.common.urn.Urn; import com.linkedin.metadata.aspect.patch.PatchOperationType; import java.util.List; -import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.commons.lang3.tuple.ImmutableTriple; @@ -17,8 +16,8 @@ public class StructuredPropertiesPatchBuilder extends AbstractMultiFieldPatchBuilder { private static final String BASE_PATH = "/properties"; - private static final String URN_KEY = "urn"; - private static final String CONTEXT_KEY = "context"; + private static final String URN_KEY = "propertyUrn"; + private static final String VALUES_KEY = "values"; /** * Remove a property from a structured properties aspect. If the property doesn't exist, this is a @@ -34,63 +33,77 @@ public StructuredPropertiesPatchBuilder removeProperty(Urn propertyUrn) { return this; } - public StructuredPropertiesPatchBuilder setProperty( - @Nonnull Urn propertyUrn, @Nullable List propertyValues) { - propertyValues.stream() - .map( - propertyValue -> - propertyValue instanceof Integer - ? this.setProperty(propertyUrn, (Integer) propertyValue) - : this.setProperty(propertyUrn, String.valueOf(propertyValue))) - .collect(Collectors.toList()); - return this; - } - - public StructuredPropertiesPatchBuilder setProperty( + public StructuredPropertiesPatchBuilder setNumberProperty( @Nonnull Urn propertyUrn, @Nullable Integer propertyValue) { - ValueNode propertyValueNode = instance.numberNode((Integer) propertyValue); - ObjectNode value = instance.objectNode(); - value.put(URN_KEY, propertyUrn.toString()); + ObjectNode newProperty = instance.objectNode(); + newProperty.put(URN_KEY, propertyUrn.toString()); + + ArrayNode valuesNode = instance.arrayNode(); + ObjectNode propertyValueNode = instance.objectNode(); + propertyValueNode.set("double", instance.numberNode(propertyValue)); + valuesNode.add(propertyValueNode); + newProperty.set(VALUES_KEY, valuesNode); + pathValues.add( ImmutableTriple.of( - PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, propertyValueNode)); + PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, newProperty)); return this; } - public StructuredPropertiesPatchBuilder setProperty( - @Nonnull Urn propertyUrn, @Nullable String propertyValue) { - ValueNode propertyValueNode = instance.textNode(String.valueOf(propertyValue)); - ObjectNode value = instance.objectNode(); - value.put(URN_KEY, propertyUrn.toString()); + public StructuredPropertiesPatchBuilder setNumberProperty( + @Nonnull Urn propertyUrn, @Nonnull List propertyValues) { + ObjectNode newProperty = instance.objectNode(); + newProperty.put(URN_KEY, propertyUrn.toString()); + + ArrayNode valuesNode = instance.arrayNode(); + propertyValues.forEach( + propertyValue -> { + ObjectNode propertyValueNode = instance.objectNode(); + propertyValueNode.set("double", instance.numberNode(propertyValue)); + valuesNode.add(propertyValueNode); + }); + newProperty.set(VALUES_KEY, valuesNode); + pathValues.add( ImmutableTriple.of( - PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, propertyValueNode)); + PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, newProperty)); return this; } - public StructuredPropertiesPatchBuilder addProperty( - @Nonnull Urn propertyUrn, @Nullable Integer propertyValue) { - ValueNode propertyValueNode = instance.numberNode((Integer) propertyValue); - ObjectNode value = instance.objectNode(); - value.put(URN_KEY, propertyUrn.toString()); + public StructuredPropertiesPatchBuilder setStringProperty( + @Nonnull Urn propertyUrn, @Nullable String propertyValue) { + ObjectNode newProperty = instance.objectNode(); + newProperty.put(URN_KEY, propertyUrn.toString()); + + ArrayNode valuesNode = instance.arrayNode(); + ObjectNode propertyValueNode = instance.objectNode(); + propertyValueNode.set("string", instance.textNode(propertyValue)); + valuesNode.add(propertyValueNode); + newProperty.set(VALUES_KEY, valuesNode); + pathValues.add( ImmutableTriple.of( - PatchOperationType.ADD.getValue(), - BASE_PATH + "/" + propertyUrn + "/" + String.valueOf(propertyValue), - propertyValueNode)); + PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, newProperty)); return this; } - public StructuredPropertiesPatchBuilder addProperty( - @Nonnull Urn propertyUrn, @Nullable String propertyValue) { - ValueNode propertyValueNode = instance.textNode(String.valueOf(propertyValue)); - ObjectNode value = instance.objectNode(); - value.put(URN_KEY, propertyUrn.toString()); + public StructuredPropertiesPatchBuilder setStringProperty( + @Nonnull Urn propertyUrn, @Nonnull List propertyValues) { + ObjectNode newProperty = instance.objectNode(); + newProperty.put(URN_KEY, propertyUrn.toString()); + + ArrayNode valuesNode = instance.arrayNode(); + propertyValues.forEach( + propertyValue -> { + ObjectNode propertyValueNode = instance.objectNode(); + propertyValueNode.set("string", instance.textNode(propertyValue)); + valuesNode.add(propertyValueNode); + }); + newProperty.set(VALUES_KEY, valuesNode); + pathValues.add( ImmutableTriple.of( - PatchOperationType.ADD.getValue(), - BASE_PATH + "/" + propertyUrn + "/" + String.valueOf(propertyValue), - propertyValueNode)); + PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, newProperty)); return this; } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertyDefinitionPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertyDefinitionPatchBuilder.java new file mode 100644 index 0000000000000..0811e2f52d003 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertyDefinitionPatchBuilder.java @@ -0,0 +1,146 @@ +package com.linkedin.metadata.aspect.patch.builder; + +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.*; + +import com.datahub.util.RecordUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.linkedin.data.template.StringArrayMap; +import com.linkedin.metadata.aspect.patch.PatchOperationType; +import com.linkedin.structured.PropertyCardinality; +import com.linkedin.structured.PropertyValue; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.commons.lang3.tuple.ImmutableTriple; + +public class StructuredPropertyDefinitionPatchBuilder + extends AbstractMultiFieldPatchBuilder { + + public static final String PATH_DELIM = "/"; + public static final String QUALIFIED_NAME_FIELD = "qualifiedName"; + public static final String DISPLAY_NAME_FIELD = "displayName"; + public static final String VALUE_TYPE_FIELD = "valueType"; + public static final String TYPE_QUALIFIER_FIELD = "typeQualifier"; + public static final String ALLOWED_VALUES_FIELD = "allowedValues"; + public static final String CARDINALITY_FIELD = "cardinality"; + public static final String ENTITY_TYPES_FIELD = "entityTypes"; + public static final String DESCRIPTION_FIELD = "description"; + public static final String IMMUTABLE_FIELD = "immutable"; + + // can only be used when creating a new structured property + public StructuredPropertyDefinitionPatchBuilder setQualifiedName(@Nonnull String name) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + QUALIFIED_NAME_FIELD, + instance.textNode(name))); + return this; + } + + public StructuredPropertyDefinitionPatchBuilder setDisplayName(@Nonnull String displayName) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + DISPLAY_NAME_FIELD, + instance.textNode(displayName))); + return this; + } + + public StructuredPropertyDefinitionPatchBuilder setValueType(@Nonnull String valueTypeUrn) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + VALUE_TYPE_FIELD, + instance.textNode(valueTypeUrn))); + return this; + } + + // can only be used when creating a new structured property + public StructuredPropertyDefinitionPatchBuilder setTypeQualifier( + @Nonnull StringArrayMap typeQualifier) { + ObjectNode value = instance.objectNode(); + typeQualifier.forEach( + (key, values) -> { + ArrayNode valuesNode = instance.arrayNode(); + values.forEach(valuesNode::add); + value.set(key, valuesNode); + }); + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), PATH_DELIM + TYPE_QUALIFIER_FIELD, value)); + return this; + } + + public StructuredPropertyDefinitionPatchBuilder addAllowedValue( + @Nonnull PropertyValue propertyValue) { + try { + ObjectNode valueNode = + (ObjectNode) new ObjectMapper().readTree(RecordUtils.toJsonString(propertyValue)); + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + ALLOWED_VALUES_FIELD + PATH_DELIM + propertyValue.getValue(), + valueNode)); + return this; + } catch (JsonProcessingException e) { + throw new IllegalArgumentException( + "Failed to add allowed value, failed to parse provided aspect json.", e); + } + } + + public StructuredPropertyDefinitionPatchBuilder setCardinality( + @Nonnull PropertyCardinality cardinality) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + CARDINALITY_FIELD, + instance.textNode(cardinality.toString()))); + return this; + } + + public StructuredPropertyDefinitionPatchBuilder addEntityType(@Nonnull String entityTypeUrn) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + ENTITY_TYPES_FIELD + PATH_DELIM + entityTypeUrn, + instance.textNode(entityTypeUrn))); + return this; + } + + public StructuredPropertyDefinitionPatchBuilder setDescription(@Nullable String description) { + if (description == null) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.REMOVE.getValue(), PATH_DELIM + DESCRIPTION_FIELD, null)); + } else { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + DESCRIPTION_FIELD, + instance.textNode(description))); + } + return this; + } + + public StructuredPropertyDefinitionPatchBuilder setImmutable(boolean immutable) { + this.pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + PATH_DELIM + IMMUTABLE_FIELD, + instance.booleanNode(immutable))); + return this; + } + + @Override + protected String getAspectName() { + return STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; + } + + @Override + protected String getEntityType() { + return STRUCTURED_PROPERTY_ENTITY_NAME; + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/ArrayMergingTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/ArrayMergingTemplate.java index ff721e97c0e1d..9c1946cd6b6d1 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/ArrayMergingTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/ArrayMergingTemplate.java @@ -12,6 +12,8 @@ public interface ArrayMergingTemplate extends Template { + static final String UNIT_SEPARATOR_DELIMITER = "␟"; + /** * Takes an Array field on the {@link RecordTemplate} subtype along with a set of key fields to * transform into a map Avoids producing side effects by copying nodes, use resulting node and not @@ -39,7 +41,15 @@ default JsonNode arrayFieldToMap( JsonNode nodeClone = node.deepCopy(); if (!keyFields.isEmpty()) { for (String keyField : keyFields) { - String key = node.get(keyField).asText(); + String key; + // if the keyField has a unit separator, we are working with a nested key + if (keyField.contains(UNIT_SEPARATOR_DELIMITER)) { + String[] keyParts = keyField.split(UNIT_SEPARATOR_DELIMITER); + JsonNode keyObject = node.get(keyParts[0]); + key = keyObject.get(keyParts[1]).asText(); + } else { + key = node.get(keyField).asText(); + } keyValue = keyValue.get(key) == null ? (ObjectNode) keyValue.set(key, instance.objectNode()).get(key) diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/AspectTemplateEngine.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/AspectTemplateEngine.java index a2dfdc73a969a..ce36b7e77a2b1 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/AspectTemplateEngine.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/AspectTemplateEngine.java @@ -13,6 +13,7 @@ import static com.linkedin.metadata.Constants.GLOSSARY_TERMS_ASPECT_NAME; import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; import static com.linkedin.metadata.Constants.UPSTREAM_LINEAGE_ASPECT_NAME; import com.fasterxml.jackson.core.JsonProcessingException; @@ -48,6 +49,7 @@ public class AspectTemplateEngine { CHART_INFO_ASPECT_NAME, DASHBOARD_INFO_ASPECT_NAME, STRUCTURED_PROPERTIES_ASPECT_NAME, + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, FORM_INFO_ASPECT_NAME) .collect(Collectors.toSet()); diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/structuredproperty/StructuredPropertyDefinitionTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/structuredproperty/StructuredPropertyDefinitionTemplate.java new file mode 100644 index 0000000000000..2a0fa4950e157 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/structuredproperty/StructuredPropertyDefinitionTemplate.java @@ -0,0 +1,86 @@ +package com.linkedin.metadata.aspect.patch.template.structuredproperty; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.linkedin.common.UrnArray; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.aspect.patch.template.CompoundKeyTemplate; +import com.linkedin.structured.StructuredPropertyDefinition; +import java.util.Collections; +import javax.annotation.Nonnull; + +public class StructuredPropertyDefinitionTemplate + extends CompoundKeyTemplate { + + private static final String ENTITY_TYPES_FIELD_NAME = "entityTypes"; + private static final String ALLOWED_VALUES_FIELD_NAME = "allowedValues"; + private static final String VALUE_FIELD_NAME = "value"; + private static final String UNIT_SEPARATOR_DELIMITER = "␟"; + + @Override + public StructuredPropertyDefinition getSubtype(RecordTemplate recordTemplate) + throws ClassCastException { + if (recordTemplate instanceof StructuredPropertyDefinition) { + return (StructuredPropertyDefinition) recordTemplate; + } + throw new ClassCastException("Unable to cast RecordTemplate to StructuredPropertyDefinition"); + } + + @Override + public Class getTemplateType() { + return StructuredPropertyDefinition.class; + } + + @Nonnull + @Override + public StructuredPropertyDefinition getDefault() { + StructuredPropertyDefinition definition = new StructuredPropertyDefinition(); + definition.setQualifiedName(""); + definition.setValueType(UrnUtils.getUrn("urn:li:dataType:datahub.string")); + definition.setEntityTypes(new UrnArray()); + + return definition; + } + + @Nonnull + @Override + public JsonNode transformFields(JsonNode baseNode) { + JsonNode transformedNode = + arrayFieldToMap(baseNode, ENTITY_TYPES_FIELD_NAME, Collections.emptyList()); + + if (transformedNode.get(ALLOWED_VALUES_FIELD_NAME) == null) { + return transformedNode; + } + + // allowedValues has a nested key - value.string or value.number depending on type. Mapping + // needs this nested key + JsonNode allowedValues = transformedNode.get(ALLOWED_VALUES_FIELD_NAME); + if (((ArrayNode) allowedValues).size() > 0) { + JsonNode allowedValue = ((ArrayNode) allowedValues).get(0); + JsonNode value = allowedValue.get(VALUE_FIELD_NAME); + String secondaryKeyName = value.fieldNames().next(); + return arrayFieldToMap( + transformedNode, + ALLOWED_VALUES_FIELD_NAME, + Collections.singletonList( + VALUE_FIELD_NAME + UNIT_SEPARATOR_DELIMITER + secondaryKeyName)); + } + + return arrayFieldToMap( + transformedNode, ALLOWED_VALUES_FIELD_NAME, Collections.singletonList(VALUE_FIELD_NAME)); + } + + @Nonnull + @Override + public JsonNode rebaseFields(JsonNode patched) { + JsonNode patchedNode = + transformedMapToArray(patched, ENTITY_TYPES_FIELD_NAME, Collections.emptyList()); + + if (patchedNode.get(ALLOWED_VALUES_FIELD_NAME) == null) { + return patchedNode; + } + return transformedMapToArray( + patchedNode, ALLOWED_VALUES_FIELD_NAME, Collections.singletonList(VALUE_FIELD_NAME)); + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java index 1fcc7149079dd..c60f89c510cd7 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java @@ -21,6 +21,7 @@ import com.linkedin.metadata.aspect.patch.template.dataset.EditableSchemaMetadataTemplate; import com.linkedin.metadata.aspect.patch.template.dataset.UpstreamLineageTemplate; import com.linkedin.metadata.aspect.patch.template.form.FormInfoTemplate; +import com.linkedin.metadata.aspect.patch.template.structuredproperty.StructuredPropertyDefinitionTemplate; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.DefaultEntitySpec; import com.linkedin.metadata.models.EntitySpec; @@ -88,6 +89,8 @@ private AspectTemplateEngine populateTemplateEngine(Map aspe aspectSpecTemplateMap.put(DATA_JOB_INPUT_OUTPUT_ASPECT_NAME, new DataJobInputOutputTemplate()); aspectSpecTemplateMap.put( STRUCTURED_PROPERTIES_ASPECT_NAME, new StructuredPropertiesTemplate()); + aspectSpecTemplateMap.put( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, new StructuredPropertyDefinitionTemplate()); aspectSpecTemplateMap.put(FORM_INFO_ASPECT_NAME, new FormInfoTemplate()); return new AspectTemplateEngine(aspectSpecTemplateMap); } diff --git a/metadata-ingestion/examples/library/dataset_add_structured_properties.py b/metadata-ingestion/examples/library/dataset_add_structured_properties.py new file mode 100644 index 0000000000000..fc2c379340592 --- /dev/null +++ b/metadata-ingestion/examples/library/dataset_add_structured_properties.py @@ -0,0 +1,24 @@ +import logging + +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.emitter.rest_emitter import DataHubRestEmitter +from datahub.specific.dataset import DatasetPatchBuilder + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +# Create rest emitter +rest_emitter = DataHubRestEmitter(gms_server="http://localhost:8080") + +dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD") + + +for patch_mcp in ( + DatasetPatchBuilder(dataset_urn) + .add_structured_property("io.acryl.dataManagement.replicationSLA", 12) + .build() +): + rest_emitter.emit(patch_mcp) + + +log.info(f"Added cluster_name, retention_time properties to dataset {dataset_urn}") diff --git a/metadata-ingestion/examples/library/dataset_remove_structured_properties.py b/metadata-ingestion/examples/library/dataset_remove_structured_properties.py new file mode 100644 index 0000000000000..d9f3c50464db2 --- /dev/null +++ b/metadata-ingestion/examples/library/dataset_remove_structured_properties.py @@ -0,0 +1,24 @@ +import logging + +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.emitter.rest_emitter import DataHubRestEmitter +from datahub.specific.dataset import DatasetPatchBuilder + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +# Create rest emitter +rest_emitter = DataHubRestEmitter(gms_server="http://localhost:8080") + +dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD") + + +for patch_mcp in ( + DatasetPatchBuilder(dataset_urn) + .remove_structured_property("io.acryl.dataManagement.replicationSLA") + .build() +): + rest_emitter.emit(patch_mcp) + + +log.info(f"Added cluster_name, retention_time properties to dataset {dataset_urn}") diff --git a/metadata-ingestion/examples/library/dataset_update_structured_properties.py b/metadata-ingestion/examples/library/dataset_update_structured_properties.py new file mode 100644 index 0000000000000..c0a2a5a9f6b6b --- /dev/null +++ b/metadata-ingestion/examples/library/dataset_update_structured_properties.py @@ -0,0 +1,24 @@ +import logging + +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.emitter.rest_emitter import DataHubRestEmitter +from datahub.specific.dataset import DatasetPatchBuilder + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +# Create rest emitter +rest_emitter = DataHubRestEmitter(gms_server="http://localhost:8080") + +dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD") + + +for patch_mcp in ( + DatasetPatchBuilder(dataset_urn) + .set_structured_property("io.acryl.dataManagement.replicationSLA", 120) + .build() +): + rest_emitter.emit(patch_mcp) + + +log.info(f"Added cluster_name, retention_time properties to dataset {dataset_urn}") diff --git a/metadata-ingestion/examples/structured_properties/create_structured_property.py b/metadata-ingestion/examples/structured_properties/create_structured_property.py new file mode 100644 index 0000000000000..e66ac3aec4122 --- /dev/null +++ b/metadata-ingestion/examples/structured_properties/create_structured_property.py @@ -0,0 +1,98 @@ +import logging + +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter + +# Imports for metadata model classes +from datahub.metadata.schema_classes import ( + PropertyValueClass, + StructuredPropertyDefinitionClass, +) +from datahub.metadata.urns import StructuredPropertyUrn + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +# Create rest emitter +rest_emitter = DatahubRestEmitter(gms_server="http://localhost:8080") + +# first, let's make an open ended structured property that allows one text value +text_property_urn = StructuredPropertyUrn("openTextProperty") +text_property_definition = StructuredPropertyDefinitionClass( + qualifiedName="io.acryl.openTextProperty", + displayName="Open Text Property", + valueType="urn:li:dataType:datahub.string", + cardinality="SINGLE", + entityTypes=[ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.container", + ], + description="This structured property allows a signle open ended response as a value", + immutable=False, +) + +event_prop_1: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityUrn=str(text_property_urn), + aspect=text_property_definition, +) +rest_emitter.emit(event_prop_1) + +# next, let's make a property that allows for multiple datahub entity urns as values +# This example property could be used to reference other users or groups in datahub +urn_property_urn = StructuredPropertyUrn("dataSteward") +urn_property_definition = StructuredPropertyDefinitionClass( + qualifiedName="io.acryl.dataManagement.dataSteward", + displayName="Data Steward", + valueType="urn:li:dataType:datahub.urn", + cardinality="MULTIPLE", + entityTypes=["urn:li:entityType:datahub.dataset"], + description="The data stewards of this asset are in charge of ensuring data cleanliness and governance", + immutable=True, + typeQualifier={ + "allowedTypes": [ + "urn:li:entityType:datahub.corpuser", + "urn:li:entityType:datahub.corpGroup", + ] + }, # this line ensures only user or group urns can be assigned as values +) + +event_prop_2: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityUrn=str(urn_property_urn), + aspect=urn_property_definition, +) +rest_emitter.emit(event_prop_2) + +# finally, let's make a single select number property with a few allowed options +number_property_urn = StructuredPropertyUrn("replicationSLA") +number_property_definition = StructuredPropertyDefinitionClass( + qualifiedName="io.acryl.dataManagement.replicationSLA", + displayName="Retention Time", + valueType="urn:li:dataType:datahub.number", + cardinality="SINGLE", + entityTypes=[ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow", + ], + description="SLA for how long data can be delayed before replicating to the destination cluster", + immutable=False, + allowedValues=[ + PropertyValueClass( + value=30, + description="30 days, usually reserved for datasets that are ephemeral and contain pii", + ), + PropertyValueClass( + value=90, + description="Use this for datasets that drive monthly reporting but contain pii", + ), + PropertyValueClass( + value=365, + description="Use this for non-sensitive data that can be retained for longer", + ), + ], +) + +event_prop_3: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityUrn=str(number_property_urn), + aspect=number_property_definition, +) +rest_emitter.emit(event_prop_3) diff --git a/metadata-ingestion/examples/structured_properties/update_structured_property.py b/metadata-ingestion/examples/structured_properties/update_structured_property.py new file mode 100644 index 0000000000000..9b80ebc236d8b --- /dev/null +++ b/metadata-ingestion/examples/structured_properties/update_structured_property.py @@ -0,0 +1,43 @@ +import logging +from typing import Union + +from datahub.configuration.kafka import KafkaProducerConnectionConfig +from datahub.emitter.kafka_emitter import DatahubKafkaEmitter, KafkaEmitterConfig +from datahub.emitter.rest_emitter import DataHubRestEmitter +from datahub.metadata.urns import StructuredPropertyUrn +from datahub.specific.structured_property import StructuredPropertyPatchBuilder + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + + +# Get an emitter, either REST or Kafka, this example shows you both +def get_emitter() -> Union[DataHubRestEmitter, DatahubKafkaEmitter]: + USE_REST_EMITTER = True + if USE_REST_EMITTER: + gms_endpoint = "http://localhost:8080" + return DataHubRestEmitter(gms_server=gms_endpoint) + else: + kafka_server = "localhost:9092" + schema_registry_url = "http://localhost:8081" + return DatahubKafkaEmitter( + config=KafkaEmitterConfig( + connection=KafkaProducerConnectionConfig( + bootstrap=kafka_server, schema_registry_url=schema_registry_url + ) + ) + ) + + +# input your unique structured property ID +property_urn = StructuredPropertyUrn("dataSteward") + +with get_emitter() as emitter: + for patch_mcp in ( + StructuredPropertyPatchBuilder(str(property_urn)) + .set_display_name("test display name") + .set_cardinality("MULTIPLE") + .add_entity_type("urn:li:entityType:datahub.dataJob") + .build() + ): + emitter.emit(patch_mcp) diff --git a/metadata-ingestion/src/datahub/cli/cli_utils.py b/metadata-ingestion/src/datahub/cli/cli_utils.py index 1bb3b01e078dd..d8939ddcff09c 100644 --- a/metadata-ingestion/src/datahub/cli/cli_utils.py +++ b/metadata-ingestion/src/datahub/cli/cli_utils.py @@ -522,6 +522,7 @@ def get_aspects_for_entity( aspects: List[str], typed: bool = False, cached_session_host: Optional[Tuple[Session, str]] = None, + details: bool = False, ) -> Dict[str, Union[dict, _Aspect]]: # Process non-timeseries aspects non_timeseries_aspects = [a for a in aspects if a not in TIMESERIES_ASPECT_MAP] @@ -553,7 +554,12 @@ def get_aspects_for_entity( aspect_name ) - aspect_dict = a["value"] + if details: + aspect_dict = a + for k in ["name", "version", "type"]: + del aspect_dict[k] + else: + aspect_dict = a["value"] if not typed: aspect_map[aspect_name] = aspect_dict elif aspect_py_class: diff --git a/metadata-ingestion/src/datahub/cli/get_cli.py b/metadata-ingestion/src/datahub/cli/get_cli.py index 6b779f8565a7f..46e2fdf5b1f79 100644 --- a/metadata-ingestion/src/datahub/cli/get_cli.py +++ b/metadata-ingestion/src/datahub/cli/get_cli.py @@ -21,10 +21,17 @@ def get() -> None: @get.command() @click.option("--urn", required=False, type=str) @click.option("-a", "--aspect", required=False, multiple=True, type=str) +@click.option( + "--details/--no-details", + required=False, + is_flag=True, + default=False, + help="Whether to print details from database which help in audit.", +) @click.pass_context @upgrade.check_upgrade @telemetry.with_telemetry() -def urn(ctx: Any, urn: Optional[str], aspect: List[str]) -> None: +def urn(ctx: Any, urn: Optional[str], aspect: List[str], details: bool) -> None: """ Get metadata for an entity with an optional list of aspects to project. This works for both versioned aspects and timeseries aspects. For timeseries aspects, it fetches the latest value. @@ -39,7 +46,9 @@ def urn(ctx: Any, urn: Optional[str], aspect: List[str]) -> None: logger.debug(f"Using urn from args {urn}") click.echo( json.dumps( - get_aspects_for_entity(entity_urn=urn, aspects=aspect, typed=False), + get_aspects_for_entity( + entity_urn=urn, aspects=aspect, typed=False, details=details + ), sort_keys=True, indent=2, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index d65d17f223361..05aa90dd76f6b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -949,9 +949,11 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: ] def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: - database_seen = set() databases, tables = self.get_all_databases_and_tables() + for database in databases.values(): + yield from self.gen_database_containers(database) + for table in tables: database_name = table["DatabaseName"] table_name = table["Name"] @@ -962,9 +964,6 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ) or not self.source_config.table_pattern.allowed(full_table_name): self.report.report_table_dropped(full_table_name) continue - if database_name not in database_seen: - database_seen.add(database_name) - yield from self.gen_database_containers(databases[database_name]) dataset_urn = make_dataset_urn_with_platform_instance( platform=self.platform, diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/operation_config.py b/metadata-ingestion/src/datahub/ingestion/source_config/operation_config.py index 3deb9e75d97e7..a670173aa4751 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_config/operation_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source_config/operation_config.py @@ -69,10 +69,10 @@ def is_profiling_enabled(operation_config: OperationConfig) -> bool: today = datetime.date.today() if ( operation_config.profile_day_of_week is not None - and operation_config.profile_date_of_month != today.weekday() + and operation_config.profile_day_of_week != today.weekday() ): logger.info( - "Profiling won't be done because weekday does not match config profile_date_of_month.", + "Profiling won't be done because weekday does not match config profile_day_of_week.", ) return False if ( diff --git a/metadata-ingestion/src/datahub/specific/structured_property.py b/metadata-ingestion/src/datahub/specific/structured_property.py new file mode 100644 index 0000000000000..50f1f079c2aa7 --- /dev/null +++ b/metadata-ingestion/src/datahub/specific/structured_property.py @@ -0,0 +1,125 @@ +from typing import Dict, List, Optional, Union + +from datahub.emitter.mcp_patch_builder import MetadataPatchProposal +from datahub.metadata.schema_classes import ( + KafkaAuditHeaderClass, + PropertyValueClass, + StructuredPropertyDefinitionClass as StructuredPropertyDefinition, + SystemMetadataClass, +) +from datahub.utilities.urns.urn import Urn + + +# This patch builder is for structured property entities. Not for the aspect on assets. +class StructuredPropertyPatchBuilder(MetadataPatchProposal): + def __init__( + self, + urn: str, + system_metadata: Optional[SystemMetadataClass] = None, + audit_header: Optional[KafkaAuditHeaderClass] = None, + ) -> None: + super().__init__( + urn, system_metadata=system_metadata, audit_header=audit_header + ) + + # can only be used when creating a new structured property + def set_qualified_name( + self, qualified_name: str + ) -> "StructuredPropertyPatchBuilder": + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/qualifiedName", + value=qualified_name, + ) + return self + + def set_display_name( + self, display_name: Optional[str] = None + ) -> "StructuredPropertyPatchBuilder": + if display_name is not None: + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/displayName", + value=display_name, + ) + return self + + # can only be used when creating a new structured property + def set_value_type( + self, value_type: Union[str, Urn] + ) -> "StructuredPropertyPatchBuilder": + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/valueType", + value=value_type, + ) + return self + + # can only be used when creating a new structured property + def set_type_qualifier( + self, type_qualifier: Optional[Dict[str, List[str]]] = None + ) -> "StructuredPropertyPatchBuilder": + if type_qualifier is not None: + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/typeQualifier", + value=type_qualifier, + ) + return self + + # can only be used when creating a new structured property + def add_allowed_value( + self, allowed_value: PropertyValueClass + ) -> "StructuredPropertyPatchBuilder": + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path=f"/allowedValues/{str(allowed_value.get('value'))}", + value=allowed_value, + ) + return self + + def set_cardinality(self, cardinality: str) -> "StructuredPropertyPatchBuilder": + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/cardinality", + value=cardinality, + ) + return self + + def add_entity_type( + self, entity_type: Union[str, Urn] + ) -> "StructuredPropertyPatchBuilder": + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path=f"/entityTypes/{self.quote(str(entity_type))}", + value=entity_type, + ) + return self + + def set_description( + self, description: Optional[str] = None + ) -> "StructuredPropertyPatchBuilder": + if description is not None: + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/description", + value=description, + ) + return self + + def set_immutable(self, immutable: bool) -> "StructuredPropertyPatchBuilder": + self._add_patch( + StructuredPropertyDefinition.ASPECT_NAME, + "add", + path="/immutable", + value=immutable, + ) + return self diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json index f180185f67ead..b72124f23d749 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json @@ -55,6 +55,59 @@ } } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "glue", + "env": "PROD", + "database": "empty-database", + "CreateTime": "June 1, 2021 at 14:55:13" + }, + "name": "empty-database", + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:glue" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { @@ -236,6 +289,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -473,6 +527,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" @@ -658,6 +713,7 @@ "type": "DATAOWNER" } ], + "ownerTypes": {}, "lastModified": { "time": 0, "actor": "urn:li:corpuser:unknown" diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json index 4b64ee1bf08d4..c66ba75548f14 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json @@ -57,6 +57,61 @@ } } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ac4381240e82d55400c22e4392e744a4", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "glue", + "instance": "some_instance_name", + "env": "PROD", + "database": "empty-database", + "CreateTime": "June 1, 2021 at 14:55:13" + }, + "name": "empty-database", + "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ac4381240e82d55400c22e4392e744a4", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ac4381240e82d55400c22e4392e744a4", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:glue", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)" + } + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ac4381240e82d55400c22e4392e744a4", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + } +}, { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { diff --git a/metadata-ingestion/tests/unit/test_glue_source.py b/metadata-ingestion/tests/unit/test_glue_source.py index 5e721fc5c1293..c8b7e021cf5a0 100644 --- a/metadata-ingestion/tests/unit/test_glue_source.py +++ b/metadata-ingestion/tests/unit/test_glue_source.py @@ -135,6 +135,11 @@ def test_glue_ingest( get_tables_response_2, {"DatabaseName": "test-database"}, ) + glue_stubber.add_response( + "get_tables", + {"TableList": []}, + {"DatabaseName": "empty-database"}, + ) glue_stubber.add_response("get_jobs", get_jobs_response, {}) glue_stubber.add_response( "get_dataflow_graph", diff --git a/metadata-ingestion/tests/unit/test_glue_source_stubs.py b/metadata-ingestion/tests/unit/test_glue_source_stubs.py index 80d16b93907f5..fc4c9e91410e0 100644 --- a/metadata-ingestion/tests/unit/test_glue_source_stubs.py +++ b/metadata-ingestion/tests/unit/test_glue_source_stubs.py @@ -77,6 +77,19 @@ ], "CatalogId": "123412341234", }, + { + "Name": "empty-database", + "CreateTime": datetime.datetime(2021, 6, 1, 14, 55, 13), + "CreateTableDefaultPermissions": [ + { + "Principal": { + "DataLakePrincipalIdentifier": "IAM_ALLOWED_PRINCIPALS" + }, + "Permissions": ["ALL"], + } + ], + "CatalogId": "123412341234", + }, ] } databases_1 = { diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java index 46d1481836fe7..95d2060079780 100644 --- a/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java @@ -17,7 +17,12 @@ import com.linkedin.common.urn.TagUrn; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.data.template.StringArrayMap; import com.linkedin.dataset.DatasetLineageType; +import com.linkedin.form.FormPrompt; +import com.linkedin.form.FormPromptType; +import com.linkedin.form.StructuredPropertyParams; import com.linkedin.metadata.aspect.patch.builder.ChartInfoPatchBuilder; import com.linkedin.metadata.aspect.patch.builder.DashboardInfoPatchBuilder; import com.linkedin.metadata.aspect.patch.builder.DataFlowInfoPatchBuilder; @@ -25,10 +30,16 @@ import com.linkedin.metadata.aspect.patch.builder.DataJobInputOutputPatchBuilder; import com.linkedin.metadata.aspect.patch.builder.DatasetPropertiesPatchBuilder; import com.linkedin.metadata.aspect.patch.builder.EditableSchemaMetadataPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.FormInfoPatchBuilder; import com.linkedin.metadata.aspect.patch.builder.OwnershipPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertiesPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder; import com.linkedin.metadata.aspect.patch.builder.UpstreamLineagePatchBuilder; import com.linkedin.metadata.graph.LineageDirection; import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PropertyCardinality; +import com.linkedin.structured.PropertyValue; import datahub.client.MetadataWriteResponse; import datahub.client.file.FileEmitter; import datahub.client.file.FileEmitterConfig; @@ -37,6 +48,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.util.Arrays; +import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import org.junit.Ignore; @@ -641,4 +653,138 @@ public void testLocalDashboardInfoRemove() { System.out.println(Arrays.asList(e.getStackTrace())); } } + + @Test + @Ignore + public void testLocalStructuredPropertyDefinitionAdd() { + RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().build()); + try { + StringArrayMap typeQualifier = new StringArrayMap(); + typeQualifier.put( + "allowedTypes", + new StringArray( + "urn:li:entityType:datahub.corpuser", "urn:li:entityType:datahub.corpGroup")); + PropertyValue propertyValue1 = new PropertyValue(); + PrimitivePropertyValue value1 = new PrimitivePropertyValue(); + value1.setString("test value 1"); + propertyValue1.setValue(value1); + PropertyValue propertyValue2 = new PropertyValue(); + PrimitivePropertyValue value2 = new PrimitivePropertyValue(); + value2.setString("test value 2"); + propertyValue2.setValue(value2); + + MetadataChangeProposal structuredPropertyDefinitionPatch = + new StructuredPropertyDefinitionPatchBuilder() + .urn(UrnUtils.getUrn("urn:li:structuredProperty:123456")) + .setQualifiedName("test.testing.123") + .setDisplayName("Test Display Name") + .setValueType("urn:li:dataType:datahub.urn") + .setTypeQualifier(typeQualifier) + .addAllowedValue(propertyValue1) + .addAllowedValue(propertyValue2) + .setCardinality(PropertyCardinality.MULTIPLE) + .addEntityType("urn:li:entityType:datahub.dataFlow") + .setDescription("test description") + .setImmutable(true) + .build(); + + Future response = restEmitter.emit(structuredPropertyDefinitionPatch); + + System.out.println(response.get().getResponseContent()); + + } catch (IOException | ExecutionException | InterruptedException e) { + System.out.println(Arrays.asList(e.getStackTrace())); + } + } + + @Test + @Ignore + public void testLocalFormInfoAdd() { + RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().build()); + try { + FormPrompt newPrompt = + new FormPrompt() + .setId("1234") + .setTitle("First Prompt") + .setType(FormPromptType.STRUCTURED_PROPERTY) + .setRequired(true) + .setStructuredPropertyParams( + new StructuredPropertyParams() + .setUrn(UrnUtils.getUrn("urn:li:structuredProperty:property1"))); + FormPrompt newPrompt2 = + new FormPrompt() + .setId("abcd") + .setTitle("Second Prompt") + .setType(FormPromptType.FIELDS_STRUCTURED_PROPERTY) + .setRequired(false) + .setStructuredPropertyParams( + new StructuredPropertyParams() + .setUrn(UrnUtils.getUrn("urn:li:structuredProperty:property1"))); + + MetadataChangeProposal formInfoPatch = + new FormInfoPatchBuilder() + .urn(UrnUtils.getUrn("urn:li:form:123456")) + .addPrompts(List.of(newPrompt, newPrompt2)) + .setName("Metadata Initiative 2024 (edited)") + .setDescription("Edited description") + .setOwnershipForm(true) + .addAssignedUser("urn:li:corpuser:admin") + .addAssignedGroup("urn:li:corpGroup:jdoe") + .build(); + Future response = restEmitter.emit(formInfoPatch); + + System.out.println(response.get().getResponseContent()); + + } catch (IOException | ExecutionException | InterruptedException e) { + System.out.println(Arrays.asList(e.getStackTrace())); + } + } + + @Test + @Ignore + public void testLocalStructuredPropertiesUpdate() { + try { + MetadataChangeProposal mcp = + new StructuredPropertiesPatchBuilder() + .urn( + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)")) + .setNumberProperty( + UrnUtils.getUrn( + "urn:li:structuredProperty:io.acryl.dataManagement.replicationSLA"), + 3456) + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + Future response = emitter.emit(mcp, null); + System.out.println(response.get().getResponseContent()); + + } catch (IOException | ExecutionException | InterruptedException e) { + System.out.println(Arrays.asList(e.getStackTrace())); + } + } + + @Test + @Ignore + public void testLocalFormInfoRemove() { + RestEmitter restEmitter = new RestEmitter(RestEmitterConfig.builder().build()); + try { + MetadataChangeProposal formInfoPatch = + new FormInfoPatchBuilder() + .urn(UrnUtils.getUrn("urn:li:form:123456")) + .removePrompts(List.of("1234", "abcd")) + .setName("Metadata Initiative 2024 (edited - again)") + .setDescription(null) + .removeAssignedUser("urn:li:corpuser:admin") + .removeAssignedGroup("urn:li:corpGroup:jdoe") + .build(); + Future response = restEmitter.emit(formInfoPatch); + + System.out.println(response.get().getResponseContent()); + + } catch (IOException | ExecutionException | InterruptedException e) { + System.out.println(Arrays.asList(e.getStackTrace())); + } + } } diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetStructuredPropertiesUpdate.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetStructuredPropertiesUpdate.java new file mode 100644 index 0000000000000..c6c15d7a068ee --- /dev/null +++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetStructuredPropertiesUpdate.java @@ -0,0 +1,79 @@ +package io.datahubproject.examples; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertiesPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class DatasetStructuredPropertiesUpdate { + + private DatasetStructuredPropertiesUpdate() {} + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + + // Adding a structured property with a single string value + MetadataChangeProposal mcp1 = + new StructuredPropertiesPatchBuilder() + .urn( + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)")) + .setStringProperty( + UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.privacy.retentionTime"), "30") + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + Future response1 = emitter.emit(mcp1, null); + System.out.println(response1.get().getResponseContent()); + + // Adding a structured property with a list of string values + List values = new ArrayList<>(); + values.add("30"); + values.add("90"); + MetadataChangeProposal mcp2 = + new StructuredPropertiesPatchBuilder() + .urn( + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)")) + .setStringProperty( + UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.privacy.retentionTime"), values) + .build(); + + Future response2 = emitter.emit(mcp2, null); + System.out.println(response2.get().getResponseContent()); + + // Adding a structured property with a single number value + MetadataChangeProposal mcp3 = + new StructuredPropertiesPatchBuilder() + .urn( + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)")) + .setNumberProperty( + UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.dataManagement.replicationSLA"), + 3456) + .build(); + + Future response3 = emitter.emit(mcp3, null); + System.out.println(response3.get().getResponseContent()); + + // Removing a structured property from a dataset + MetadataChangeProposal mcp4 = + new StructuredPropertiesPatchBuilder() + .urn( + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleCypressHiveDataset,PROD)")) + .removeProperty( + UrnUtils.getUrn("urn:li:structuredProperty:io.acryl.dataManagement.replicationSLA")) + .build(); + + Future response4 = emitter.emit(mcp4, null); + System.out.println(response4.get().getResponseContent()); + } +} diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/FormCreate.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/FormCreate.java new file mode 100644 index 0000000000000..5451d431b3b99 --- /dev/null +++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/FormCreate.java @@ -0,0 +1,68 @@ +package io.datahubproject.examples; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.form.FormActorAssignment; +import com.linkedin.form.FormInfo; +import com.linkedin.form.FormPrompt; +import com.linkedin.form.FormPromptArray; +import com.linkedin.form.FormPromptType; +import com.linkedin.form.FormType; +import com.linkedin.form.StructuredPropertyParams; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import datahub.event.MetadataChangeProposalWrapper; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class FormCreate { + + private FormCreate() {} + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + FormPromptArray prompts = new FormPromptArray(); + FormPrompt prompt1 = + new FormPrompt() + .setId("1") + .setTitle("First Prompt") + .setType(FormPromptType.STRUCTURED_PROPERTY) + .setRequired(true) + .setStructuredPropertyParams( + new StructuredPropertyParams() + .setUrn(UrnUtils.getUrn("urn:li:structuredProperty:property1"))); + FormPrompt prompt2 = + new FormPrompt() + .setId("2") + .setTitle("Second Prompt") + .setType(FormPromptType.FIELDS_STRUCTURED_PROPERTY) + .setRequired(false) + .setStructuredPropertyParams( + new StructuredPropertyParams() + .setUrn(UrnUtils.getUrn("urn:li:structuredProperty:property1"))); + prompts.add(prompt1); + prompts.add(prompt2); + + FormInfo formInfo = + new FormInfo() + .setName("Metadata Initiative 2024") + .setDescription("Please respond to this form for metadata compliance purposes.") + .setType(FormType.VERIFICATION) + .setPrompts(prompts) + .setActors(new FormActorAssignment().setOwners(true)); + + MetadataChangeProposalWrapper mcpw = + MetadataChangeProposalWrapper.builder() + .entityType("form") + .entityUrn("urn:li:form:metadata_initiative_1") + .upsert() + .aspect(formInfo) + .aspectName("formInfo") + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + Future response = emitter.emit(mcpw, null); + System.out.println(response.get().getResponseContent()); + } +} diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/FormUpdate.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/FormUpdate.java new file mode 100644 index 0000000000000..d986f8d403ef6 --- /dev/null +++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/FormUpdate.java @@ -0,0 +1,57 @@ +package io.datahubproject.examples; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.form.FormPrompt; +import com.linkedin.form.FormPromptType; +import com.linkedin.form.StructuredPropertyParams; +import com.linkedin.metadata.aspect.patch.builder.FormInfoPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class FormUpdate { + + private FormUpdate() {} + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + FormPrompt newPrompt = + new FormPrompt() + .setId("1234") + .setTitle("First Prompt") + .setType(FormPromptType.STRUCTURED_PROPERTY) + .setRequired(true) + .setStructuredPropertyParams( + new StructuredPropertyParams() + .setUrn(UrnUtils.getUrn("urn:li:structuredProperty:property1"))); + FormPrompt newPrompt2 = + new FormPrompt() + .setId("abcd") + .setTitle("Second Prompt") + .setType(FormPromptType.FIELDS_STRUCTURED_PROPERTY) + .setRequired(false) + .setStructuredPropertyParams( + new StructuredPropertyParams() + .setUrn(UrnUtils.getUrn("urn:li:structuredProperty:property1"))); + + Urn formUrn = UrnUtils.getUrn("urn:li:form:metadata_initiative_1"); + FormInfoPatchBuilder formInfoPatchBuilder = + new FormInfoPatchBuilder() + .urn(formUrn) + .addPrompts(List.of(newPrompt, newPrompt2)) + .setName("Metadata Initiative 2024 (edited)") + .setDescription("Edited description") + .setOwnershipForm(true); + MetadataChangeProposal mcp = formInfoPatchBuilder.build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + Future response = emitter.emit(mcp, null); + System.out.println(response.get().getResponseContent()); + } +} diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/StructuredPropertyUpsert.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/StructuredPropertyUpsert.java new file mode 100644 index 0000000000000..6d6bd232bf0bc --- /dev/null +++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/StructuredPropertyUpsert.java @@ -0,0 +1,102 @@ +package io.datahubproject.examples; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.data.template.StringArrayMap; +import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PropertyCardinality; +import com.linkedin.structured.PropertyValue; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class StructuredPropertyUpsert { + + private StructuredPropertyUpsert() {} + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + // open ended string structured property on datasets and dataFlows + MetadataChangeProposal mcp1 = + new StructuredPropertyDefinitionPatchBuilder() + .urn( + UrnUtils.getUrn( + "urn:li:structuredProperty:testString")) // use existing urn for update, new urn + // for new property + .setQualifiedName("io.acryl.testString") + .setDisplayName("Open Ended String") + .setValueType("urn:li:dataType:datahub.string") + .setCardinality(PropertyCardinality.SINGLE) + .addEntityType("urn:li:entityType:datahub.dataset") + .addEntityType("urn:li:entityType:datahub.dataFlow") + .setDescription("test description for open ended string") + .setImmutable(true) + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + Future response1 = emitter.emit(mcp1, null); + System.out.println(response1.get().getResponseContent()); + + // Next, let's make a property that allows for multiple datahub entity urns as values + // This example property could be used to reference other users or groups in datahub + StringArrayMap typeQualifier = new StringArrayMap(); + typeQualifier.put( + "allowedTypes", + new StringArray( + "urn:li:entityType:datahub.corpuser", "urn:li:entityType:datahub.corpGroup")); + + MetadataChangeProposal mcp2 = + new StructuredPropertyDefinitionPatchBuilder() + .urn(UrnUtils.getUrn("urn:li:structuredProperty:dataSteward")) + .setQualifiedName("io.acryl.dataManagement.dataSteward") + .setDisplayName("Data Steward") + .setValueType("urn:li:dataType:datahub.urn") + .setTypeQualifier(typeQualifier) + .setCardinality(PropertyCardinality.MULTIPLE) + .addEntityType("urn:li:entityType:datahub.dataset") + .setDescription( + "The data stewards of this asset are in charge of ensuring data cleanliness and governance") + .setImmutable(true) + .build(); + + Future response2 = emitter.emit(mcp2, null); + System.out.println(response2.get().getResponseContent()); + + // Finally, let's make a single select number property with a few allowed options + PropertyValue propertyValue1 = new PropertyValue(); + PrimitivePropertyValue value1 = new PrimitivePropertyValue(); + value1.setDouble(30.0); + propertyValue1.setDescription( + "30 days, usually reserved for datasets that are ephemeral and contain pii"); + propertyValue1.setValue(value1); + PropertyValue propertyValue2 = new PropertyValue(); + PrimitivePropertyValue value2 = new PrimitivePropertyValue(); + value2.setDouble(90.0); + propertyValue2.setDescription( + "Use this for datasets that drive monthly reporting but contain pii"); + propertyValue2.setValue(value2); + + MetadataChangeProposal mcp3 = + new StructuredPropertyDefinitionPatchBuilder() + .urn(UrnUtils.getUrn("urn:li:structuredProperty:replicationSLA")) + .setQualifiedName("io.acryl.dataManagement.replicationSLA") + .setDisplayName("Replication SLA") + .setValueType("urn:li:dataType:datahub.string") + .addAllowedValue(propertyValue1) + .addAllowedValue(propertyValue2) + .setCardinality(PropertyCardinality.SINGLE) + .addEntityType("urn:li:entityType:datahub.dataset") + .setDescription( + "SLA for how long data can be delayed before replicating to the destination cluster") + .setImmutable(false) + .build(); + + Future response3 = emitter.emit(mcp3, null); + System.out.println(response3.get().getResponseContent()); + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java index 30988d81db2f9..dd5a81a0d81ec 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java @@ -147,7 +147,9 @@ public ResponseEntity getEntities( @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") Boolean withSystemMetadata, @RequestParam(value = "skipCache", required = false, defaultValue = "false") - Boolean skipCache) + Boolean skipCache, + @RequestParam(value = "includeSoftDelete", required = false, defaultValue = "false") + Boolean includeSoftDelete) throws URISyntaxException { EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); @@ -175,7 +177,8 @@ public ResponseEntity getEntities( searchService.scrollAcrossEntities( opContext .withSearchFlags(flags -> DEFAULT_SEARCH_FLAGS) - .withSearchFlags(flags -> flags.setSkipCache(skipCache)), + .withSearchFlags(flags -> flags.setSkipCache(skipCache)) + .withSearchFlags(flags -> flags.setIncludeSoftDeleted(includeSoftDelete)), List.of(entitySpec.getName()), query, null, diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java index 93ad502c9f7d6..7c94ccf630a78 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java @@ -57,6 +57,7 @@ public class OpenAPIV3Generator { private static final String ASPECT_RESPONSE_SUFFIX = "Aspect" + RESPONSE_SUFFIX; private static final String ENTITY_REQUEST_SUFFIX = "Entity" + REQUEST_SUFFIX; private static final String ENTITY_RESPONSE_SUFFIX = "Entity" + RESPONSE_SUFFIX; + private static final String NAME_SKIP_CACHE = "skipCache"; public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) { final Set aspectNames = entityRegistry.getAspectSpecs().keySet(); @@ -249,9 +250,19 @@ private static PathItem buildListEntityPath(final EntitySpec entity) { List.of( new Parameter() .in(NAME_QUERY) - .name("systemMetadata") + .name(NAME_SYSTEM_METADATA) .description("Include systemMetadata with response.") .schema(new Schema().type(TYPE_BOOLEAN)._default(false)), + new Parameter() + .in(NAME_QUERY) + .name(NAME_INCLUDE_SOFT_DELETE) + .description("Include soft-deleted aspects with response.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(false)), + new Parameter() + .in(NAME_QUERY) + .name(NAME_SKIP_CACHE) + .description("Skip cache when listing entities.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(false)), new Parameter() .$ref( String.format( diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/FormService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/FormService.java index a75c399300f65..210fba82eb4e2 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/service/FormService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/FormService.java @@ -33,7 +33,9 @@ import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.OwnershipUtils; import com.linkedin.metadata.entity.AspectUtils; +import com.linkedin.metadata.key.FormKey; import com.linkedin.metadata.service.util.SearchBasedFormAssignmentRunner; +import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.FormUtils; import com.linkedin.metadata.utils.SchemaFieldUtils; import com.linkedin.mxe.MetadataChangeProposal; @@ -51,6 +53,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -1050,6 +1053,28 @@ private void verifyEntityExists(@Nonnull OperationContext opContext, @Nonnull fi } } + /** Create a form given the formInfo aspect. */ + public Urn createForm( + @Nonnull OperationContext opContext, + @Nonnull final FormInfo formInfo, + @Nullable final String id) { + + FormKey formKey = new FormKey(); + String formId = id != null ? id : UUID.randomUUID().toString(); + formKey.setId(formId); + Urn formUrn = EntityKeyUtils.convertEntityKeyToUrn(formKey, FORM_ENTITY_NAME); + + try { + this.entityClient.ingestProposal( + opContext, + AspectUtils.buildMetadataChangeProposal(formUrn, FORM_INFO_ASPECT_NAME, formInfo), + false); + return formUrn; + } catch (Exception e) { + throw new RuntimeException("Failed to create form", e); + } + } + private AuditStamp createSystemAuditStamp() { return createAuditStamp(UrnUtils.getUrn(SYSTEM_ACTOR)); } diff --git a/metadata-service/war/src/main/resources/boot/policies.json b/metadata-service/war/src/main/resources/boot/policies.json index 454f0ba7d1163..53c773d130f32 100644 --- a/metadata-service/war/src/main/resources/boot/policies.json +++ b/metadata-service/war/src/main/resources/boot/policies.json @@ -34,7 +34,9 @@ "MANAGE_GLOBAL_OWNERSHIP_TYPES", "GET_ANALYTICS_PRIVILEGE", "CREATE_BUSINESS_ATTRIBUTE", - "MANAGE_BUSINESS_ATTRIBUTE" + "MANAGE_BUSINESS_ATTRIBUTE", + "MANAGE_STRUCTURED_PROPERTIES", + "MANAGE_DOCUMENTATION_FORMS" ], "displayName":"Root User - All Platform Privileges", "description":"Grants all platform privileges to root user.", @@ -179,7 +181,9 @@ "MANAGE_GLOBAL_OWNERSHIP_TYPES", "GET_ANALYTICS_PRIVILEGE", "CREATE_BUSINESS_ATTRIBUTE", - "MANAGE_BUSINESS_ATTRIBUTE" + "MANAGE_BUSINESS_ATTRIBUTE", + "MANAGE_STRUCTURED_PROPERTIES", + "MANAGE_DOCUMENTATION_FORMS" ], "displayName":"Admins - Platform Policy", "description":"Admins have all platform privileges.", @@ -265,7 +269,9 @@ "MANAGE_GLOBAL_ANNOUNCEMENTS", "MANAGE_GLOSSARIES", "MANAGE_TAGS", - "MANAGE_BUSINESS_ATTRIBUTE" + "MANAGE_BUSINESS_ATTRIBUTE", + "MANAGE_STRUCTURED_PROPERTIES", + "MANAGE_DOCUMENTATION_FORMS" ], "displayName":"Editors - Platform Policy", "description":"Editors can manage ingestion and view analytics.", diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java index 24fa4ec080cfa..a4fcb65687353 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java @@ -151,6 +151,18 @@ public class PoliciesConfig { "Manage Connections", "Manage connections to external DataHub platforms."); + public static final Privilege MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE = + Privilege.of( + "MANAGE_STRUCTURED_PROPERTIES", + "Manage Structured Properties", + "Manage structured properties in your instance."); + + public static final Privilege MANAGE_DOCUMENTATION_FORMS_PRIVILEGE = + Privilege.of( + "MANAGE_DOCUMENTATION_FORMS", + "Manage Documentation Forms", + "Manage forms assigned to assets to assist in documentation efforts."); + public static final List PLATFORM_PRIVILEGES = ImmutableList.of( MANAGE_POLICIES_PRIVILEGE, @@ -175,7 +187,9 @@ public class PoliciesConfig { MANAGE_GLOBAL_OWNERSHIP_TYPES, CREATE_BUSINESS_ATTRIBUTE_PRIVILEGE, MANAGE_BUSINESS_ATTRIBUTE_PRIVILEGE, - MANAGE_CONNECTIONS_PRIVILEGE); + MANAGE_CONNECTIONS_PRIVILEGE, + MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE, + MANAGE_DOCUMENTATION_FORMS_PRIVILEGE); // Resource Privileges //