Skip to content

Commit

Permalink
Fixes open-metadata#14006 Validate tag labels before creating/updatin…
Browse files Browse the repository at this point in the history 
…g an entity (open-metadata#14027)
  • Loading branch information
@sureshms
sureshms authored Nov 19, 2023
1 parent dbecf6d commit c4d6f68
Show file tree
Hide file tree
Showing 19 changed files with 448 additions and 125 deletions.
49 changes: 9 additions & 40 deletions openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package org.openmetadata.service.jdbi3;

import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.schema.type.Include.ALL;
import static org.openmetadata.service.Entity.CONTAINER;
import static org.openmetadata.service.Entity.DASHBOARD_DATA_MODEL;
Expand Down Expand Up @@ -113,11 +112,6 @@ public void prepare(Container container, boolean update) {
Container parent = Entity.getEntity(container.getParent(), "owner", ALL);
container.withParent(parent.getEntityReference());
}
// Validate field tags
if (container.getDataModel() != null) {
addDerivedColumnTags(container.getDataModel().getColumns());
validateColumnTags(container.getDataModel().getColumns());
}
}

@Override
Expand Down Expand Up @@ -173,23 +167,21 @@ public void applyTags(Container container) {
// Add container level tags by adding tag to container relationship
super.applyTags(container);
if (container.getDataModel() != null) {
applyTags(container.getDataModel().getColumns());
applyColumnTags(container.getDataModel().getColumns());
}
}

@Override
public EntityInterface getParentEntity(Container entity, String fields) {
return Entity.getEntity(entity.getService(), fields, Include.NON_DELETED);
public void validateTags(Container container) {
super.validateTags(container);
if (container.getDataModel() != null) {
validateColumnTags(container.getDataModel().getColumns());
}
}

private void applyTags(List<Column> columns) {
// Add column level tags by adding tag to column relationship
for (Column column : columns) {
applyTags(column.getTags(), column.getFullyQualifiedName());
if (column.getChildren() != null) {
applyTags(column.getChildren());
}
}
@Override
public EntityInterface getParentEntity(Container entity, String fields) {
return Entity.getEntity(entity.getService(), fields, Include.NON_DELETED);
}

@Override
Expand Down Expand Up @@ -259,29 +251,6 @@ public EntityInterface performTask(String user, ResolveTask resolveTask) {
}
}

private void addDerivedColumnTags(List<Column> columns) {
if (nullOrEmpty(columns)) {
return;
}

for (Column column : columns) {
column.setTags(addDerivedTags(column.getTags()));
if (column.getChildren() != null) {
addDerivedColumnTags(column.getChildren());
}
}
}

private void validateColumnTags(List<Column> columns) {
// Add column level tags by adding tag to column relationship
for (Column column : columns) {
checkMutuallyExclusive(column.getTags());
if (column.getChildren() != null) {
validateColumnTags(column.getChildren());
}
}
}

/** Handles entity updated from PUT and POST operations */
public class ContainerUpdater extends ColumnEntityUpdater {
public ContainerUpdater(Container original, Container updated, Operation operation) {
Expand Down
26 changes: 5 additions & 21 deletions ...ta-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,9 +122,6 @@ public void prepare(DashboardDataModel dashboardDataModel, boolean update) {
DashboardService dashboardService = Entity.getEntity(dashboardDataModel.getService(), "", Include.ALL);
dashboardDataModel.setService(dashboardService.getEntityReference());
dashboardDataModel.setServiceType(dashboardService.getServiceType());

// Validate column tags
validateColumnTags(dashboardDataModel.getColumns());
}

@Override
Expand Down Expand Up @@ -178,21 +175,11 @@ public void restorePatchAttributes(DashboardDataModel original, DashboardDataMod
updated.withService(original.getService());
}

private void applyTags(List<Column> columns) {
// Add column level tags by adding tag to column relationship
for (Column column : columns) {
applyTags(column.getTags(), column.getFullyQualifiedName());
if (column.getChildren() != null) {
applyTags(column.getChildren());
}
}
}

@Override
public void applyTags(DashboardDataModel dashboardDataModel) {
// Add table level tags by adding tag to table relationship
super.applyTags(dashboardDataModel);
applyTags(dashboardDataModel.getColumns());
applyColumnTags(dashboardDataModel.getColumns());
}

@Override
Expand All @@ -205,13 +192,10 @@ public EntityUpdater getUpdater(DashboardDataModel original, DashboardDataModel
return new DataModelUpdater(original, updated, operation);
}

private void validateColumnTags(List<Column> columns) {
for (Column column : columns) {
checkMutuallyExclusive(column.getTags());
if (column.getChildren() != null) {
validateColumnTags(column.getChildren());
}
}
@Override
public void validateTags(DashboardDataModel entity) {
super.validateTags(entity);
validateColumnTags(entity.getColumns());
}

public class DataModelUpdater extends ColumnEntityUpdater {
Expand Down
56 changes: 39 additions & 17 deletions openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,6 @@
import org.openmetadata.schema.api.VoteRequest;
import org.openmetadata.schema.api.feed.ResolveTask;
import org.openmetadata.schema.api.teams.CreateTeam;
import org.openmetadata.schema.entity.classification.Tag;
import org.openmetadata.schema.entity.data.GlossaryTerm;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.entity.teams.Team;
import org.openmetadata.schema.entity.teams.User;
Expand All @@ -116,7 +114,6 @@
import org.openmetadata.schema.type.ProviderType;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.schema.type.TagLabel.TagSource;
import org.openmetadata.schema.type.TaskType;
import org.openmetadata.schema.type.ThreadType;
import org.openmetadata.schema.type.Votes;
Expand Down Expand Up @@ -447,6 +444,7 @@ public final T copy(T entity, CreateEntity request, String updatedBy) {
entity.setDescription(request.getDescription());
entity.setOwner(owner);
entity.setDomain(domain);
entity.setTags(request.getTags());
entity.setDataProducts(getEntityReferences(Entity.DATA_PRODUCT, request.getDataProducts()));
entity.setLifeCycle(request.getLifeCycle());
entity.setExtension(request.getExtension());
Expand Down Expand Up @@ -692,10 +690,7 @@ public final T createInternal(T entity) {
}

public void prepareInternal(T entity, boolean update) {
if (supportsTags) {
entity.setTags(addDerivedTags(entity.getTags()));
checkMutuallyExclusive(entity.getTags());
}
validateTags(entity);
prepare(entity, update);
setFullyQualifiedName(entity);
validateExtension(entity);
Expand Down Expand Up @@ -1252,6 +1247,16 @@ private List<TagLabel> getDerivedTags(TagLabel tagLabel) {
return Collections.emptyList();
}

protected void applyColumnTags(List<Column> columns) {
// Add column level tags by adding tag to column relationship
for (Column column : columns) {
applyTags(column.getTags(), column.getFullyQualifiedName());
if (column.getChildren() != null) {
applyColumnTags(column.getChildren());
}
}
}

protected void applyTags(T entity) {
if (supportsTags) {
// Add entity level tags by adding tag to the entity relationship
Expand All @@ -1263,16 +1268,6 @@ protected void applyTags(T entity) {
@Transaction
public void applyTags(List<TagLabel> tagLabels, String targetFQN) {
for (TagLabel tagLabel : listOrEmpty(tagLabels)) {
if (tagLabel.getSource() == TagSource.CLASSIFICATION) {
Tag tag = daoCollection.tagDAO().findEntityByName(tagLabel.getTagFQN());
tagLabel.withDescription(tag.getDescription());
tagLabel.setSource(TagSource.CLASSIFICATION);
} else if (tagLabel.getSource() == TagLabel.TagSource.GLOSSARY) {
GlossaryTerm term = daoCollection.glossaryTermDAO().findEntityByName(tagLabel.getTagFQN(), NON_DELETED);
tagLabel.withDescription(term.getDescription());
tagLabel.setSource(TagLabel.TagSource.GLOSSARY);
}

// Apply tagLabel to targetFQN that identifies an entity or field
daoCollection
.tagUsageDAO()
Expand Down Expand Up @@ -1737,6 +1732,21 @@ public EntityReference validateOwner(EntityReference owner) {
return Entity.getEntityReferenceById(owner.getType(), owner.getId(), ALL);
}

public void validateTags(T entity) {
if (!supportsTags) {
return;
}
validateTags(entity.getTags());
entity.setTags(addDerivedTags(entity.getTags()));
checkMutuallyExclusive(entity.getTags());
}

public void validateTags(List<TagLabel> labels) {
for (TagLabel label : listOrEmpty(labels)) {
TagLabelUtil.applyTagCommonFields(label);
}
}

public EntityReference validateDomain(String domainFqn) {
if (!supportsDomain || domainFqn == null) {
return null;
Expand Down Expand Up @@ -1777,6 +1787,18 @@ public void validateTaskThread(ThreadContext threadContext) {
}
}

protected void validateColumnTags(List<Column> columns) {
// Add column level tags by adding tag to column relationship
for (Column column : listOrEmpty(columns)) {
validateTags(column.getTags());
column.setTags(addDerivedTags(column.getTags()));
checkMutuallyExclusive(column.getTags());
if (column.getChildren() != null) {
validateColumnTags(column.getChildren());
}
}
}

public enum Operation {
PUT,
PATCH,
Expand Down
19 changes: 13 additions & 6 deletions openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,9 +217,6 @@ public void restorePatchAttributes(Pipeline original, Pipeline updated) {
@Override
public void prepare(Pipeline pipeline, boolean update) {
populateService(pipeline);
if (pipeline.getTasks() != null) {
pipeline.getTasks().forEach(task -> checkMutuallyExclusive(task.getTags()));
}
}

@Override
Expand All @@ -245,15 +242,25 @@ public void storeRelationships(Pipeline pipeline) {
public void applyTags(Pipeline pipeline) {
// Add table level tags by adding tag to table relationship
super.applyTags(pipeline);
applyTags(pipeline.getTasks());
applyTaskTags(pipeline.getTasks()); // TODO need cleanup
}

@Override
public EntityInterface getParentEntity(Pipeline entity, String fields) {
return Entity.getEntity(entity.getService(), fields, Include.NON_DELETED);
}

private void applyTags(List<Task> tasks) {
@Override
public void validateTags(Pipeline entity) {
super.validateTags(entity);
for (Task task : listOrEmpty(entity.getTasks())) {
validateTags(task.getTags());
task.setTags(addDerivedTags(task.getTags()));
checkMutuallyExclusive(task.getTags());
}
}

private void applyTaskTags(List<Task> tasks) {
for (Task task : listOrEmpty(tasks)) {
applyTags(task.getTags(), task.getFullyQualifiedName());
}
Expand Down Expand Up @@ -374,7 +381,7 @@ private void updateTasks(Pipeline original, Pipeline updated) {
List<Task> added = new ArrayList<>();
List<Task> deleted = new ArrayList<>();
recordListChange(TASKS_FIELD, origTasks, updatedTasks, added, deleted, taskMatch);
applyTags(added);
applyTaskTags(added);
deleted.forEach(d -> daoCollection.tagUsageDAO().deleteTagsByTarget(d.getFullyQualifiedName()));
}
}
Expand Down
21 changes: 12 additions & 9 deletions openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,6 @@ public void prepare(SearchIndex searchIndex, boolean update) {
SearchService searchService = Entity.getEntity(searchIndex.getService(), "", ALL);
searchIndex.setService(searchService.getEntityReference());
searchIndex.setServiceType(searchService.getServiceType());
// Validate field tags
if (searchIndex.getFields() != null) {
addDerivedFieldTags(searchIndex.getFields());
validateSchemaFieldTags(searchIndex.getFields());
}
}

@Override
Expand Down Expand Up @@ -227,22 +222,30 @@ private SearchIndexField cloneWithoutTags(SearchIndexField field) {
.withChildren(children);
}

@Override
public void validateTags(SearchIndex entity) {
super.validateTags(entity);
validateSchemaFieldTags(entity.getFields());
}

private void validateSchemaFieldTags(List<SearchIndexField> fields) {
// Add field level tags by adding tag to field relationship
for (SearchIndexField field : fields) {
for (SearchIndexField field : listOrEmpty(fields)) {
validateTags(field.getTags());
field.setTags(addDerivedTags(field.getTags()));
checkMutuallyExclusive(field.getTags());
if (field.getChildren() != null) {
validateSchemaFieldTags(field.getChildren());
}
}
}

private void applyTags(List<SearchIndexField> fields) {
private void applyFieldTags(List<SearchIndexField> fields) {
// Add field level tags by adding tag to field relationship
for (SearchIndexField field : fields) {
applyTags(field.getTags(), field.getFullyQualifiedName());
if (field.getChildren() != null) {
applyTags(field.getChildren());
applyFieldTags(field.getChildren());
}
}
}
Expand All @@ -252,7 +255,7 @@ public void applyTags(SearchIndex searchIndex) {
// Add table level tags by adding tag to table relationship
super.applyTags(searchIndex);
if (searchIndex.getFields() != null) {
applyTags(searchIndex.getFields());
applyFieldTags(searchIndex.getFields());
}
}

Expand Down
Loading

0 comments on commit c4d6f68

Please sign in to comment.