From b61c38ab0539f546b65f3122962a0f84d215e581 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Sat, 30 Sep 2023 22:47:59 -0500 Subject: [PATCH] =?UTF-8?q?refactor(misc):=20testngJava=20fix,=20systemres?= =?UTF-8?q?tli=20client,=20cache=20key=20fix,=20e=E2=80=A6=20(#8926)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 2 +- .../system/elasticsearch/util/IndexUtils.java | 2 +- ...pgradeCliApplicationTestConfiguration.java | 4 + .../linkedin/metadata/entity/AspectDao.java | 4 + .../metadata/entity/EntityServiceImpl.java | 9 +- .../entity/cassandra/CassandraAspectDao.java | 7 ++ .../metadata/entity/ebean/EbeanAspectDao.java | 13 +++ .../elastic/ElasticSearchGraphService.java | 4 +- .../elasticsearch/ElasticSearchService.java | 4 +- .../indexbuilder/ESIndexBuilder.java | 27 ++++-- .../indexbuilder/EntityIndexBuilder.java | 35 -------- .../indexbuilder/EntityIndexBuilders.java | 59 +++++++------ .../indexbuilder/MappingsBuilder.java | 86 +++++++++++-------- .../indexbuilder/ReindexConfig.java | 24 ++++-- .../SearchDocumentTransformer.java | 23 +++-- .../service/UpdateIndicesService.java | 50 ++++++++--- .../metadata/shared/ElasticSearchIndexed.java | 2 +- .../ElasticSearchSystemMetadataService.java | 4 +- .../ElasticSearchTimeseriesAspectService.java | 5 +- .../TimeseriesAspectIndexBuilders.java | 5 +- .../entity/EbeanAspectMigrationsDaoTest.java | 31 +++++-- .../io/datahubproject/test/DataGenerator.java | 22 ++++- .../src/main/resources/application.properties | 2 +- ...eConsumerApplicationTestConfiguration.java | 4 + .../kafka/MetadataChangeLogProcessor.java | 7 +- .../kafka/hook/MetadataChangeLogHook.java | 8 ++ .../kafka/hook/UpdateIndicesHook.java | 2 +- .../kafka/hook/UpdateIndicesHookTest.java | 15 +++- .../spring/MCLSpringTestConfiguration.java | 4 + ...eConsumerApplicationTestConfiguration.java | 4 + .../src/main/resources/application.yml | 2 +- .../factory/entity/EntityServiceFactory.java | 18 ++-- .../entity/JavaEntityClientFactory.java | 9 +- .../indices/UpdateIndicesServiceFactory.java | 28 +++++- .../search/ElasticSearchServiceFactory.java | 7 +- .../search/EntityIndexBuildersFactory.java | 35 ++++++++ .../entity/client/EntityClientCache.java | 12 ++- .../metadata/entity/EntityService.java | 8 ++ 38 files changed, 404 insertions(+), 183 deletions(-) delete mode 100644 metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilder.java create mode 100644 metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/EntityIndexBuildersFactory.java diff --git a/build.gradle b/build.gradle index 0a94991b131aac..c8892045a6683e 100644 --- a/build.gradle +++ b/build.gradle @@ -291,7 +291,7 @@ subprojects { maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 if (project.configurations.getByName("testImplementation").getDependencies() - .any{ it.getName() == "testng" }) { + .any{ it.getName().contains("testng") }) { useTestNG() } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java index 4b04feac62cbf3..d9788448444eda 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java @@ -31,7 +31,7 @@ public static List getAllReindexConfigs(List reindexConfigs = new ArrayList<>(_reindexConfigs); if (reindexConfigs.isEmpty()) { for (ElasticSearchIndexed elasticSearchIndexed : elasticSearchIndexedList) { - reindexConfigs.addAll(elasticSearchIndexed.getReindexConfigs()); + reindexConfigs.addAll(elasticSearchIndexed.buildReindexConfigs()); } _reindexConfigs = new ArrayList<>(reindexConfigs); } diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java index b1bdead58a72b5..6cc853b2c7c4d5 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java @@ -6,6 +6,7 @@ import com.linkedin.metadata.models.registry.ConfigEntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; import io.ebean.Database; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; @@ -35,4 +36,7 @@ public class UpgradeCliApplicationTestConfiguration { @MockBean ConfigEntityRegistry configEntityRegistry; + + @MockBean + public EntityIndexBuilders entityIndexBuilders; } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java index 2d5c5e23ae5281..42dd3f0405a6af 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/AspectDao.java @@ -8,6 +8,7 @@ import io.ebean.PagedList; import io.ebean.Transaction; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.sql.Timestamp; @@ -103,6 +104,9 @@ Integer countAspect( @Nonnull PagedList getPagedAspects(final RestoreIndicesArgs args); + @Nonnull + Stream streamAspects(String entityName, String aspectName); + int deleteUrn(@Nullable Transaction tx, @Nonnull final String urn); @Nonnull diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index 66188473b9d030..57f88e31deea52 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -3,6 +3,7 @@ import com.codahale.metrics.Timer; import com.linkedin.data.template.GetMode; import com.linkedin.data.template.SetMode; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.config.PreProcessHooks; import com.datahub.util.RecordUtils; import com.datahub.util.exception.ModelConversionException; @@ -93,6 +94,7 @@ import javax.persistence.EntityNotFoundException; import io.ebean.Transaction; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import static com.linkedin.metadata.Constants.*; @@ -144,11 +146,11 @@ public class EntityServiceImpl implements EntityService { private final Map> _entityToValidAspects; private RetentionService _retentionService; private final Boolean _alwaysEmitChangeLog; + @Getter private final UpdateIndicesService _updateIndicesService; private final PreProcessHooks _preProcessHooks; protected static final int MAX_KEYS_PER_QUERY = 500; - private final Integer ebeanMaxTransactionRetry; public EntityServiceImpl( @@ -180,6 +182,11 @@ public EntityServiceImpl( ebeanMaxTransactionRetry = retry != null ? retry : DEFAULT_MAX_TRANSACTION_RETRY; } + @Override + public void setSystemEntityClient(SystemEntityClient systemEntityClient) { + this._updateIndicesService.setSystemEntityClient(systemEntityClient); + } + /** * Retrieves the latest aspects corresponding to a batch of {@link Urn}s based on a provided * set of aspect names. diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java index b215dd4a5d1ed6..9f4a36efb45013 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java @@ -41,6 +41,7 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -445,6 +446,12 @@ public PagedList getPagedAspects(final RestoreIndicesArgs args) { return null; } + @Nonnull + @Override + public Stream streamAspects(String entityName, String aspectName) { + // Not implemented + return null; + } @Override @Nonnull diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java index 30886db2649940..c16c98b34f3ebc 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java @@ -42,6 +42,7 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -433,6 +434,18 @@ public PagedList getPagedAspects(final RestoreIndicesArgs args) { .findPagedList(); } + @Override + @Nonnull + public Stream streamAspects(String entityName, String aspectName) { + ExpressionList exp = _server.find(EbeanAspectV2.class) + .select(EbeanAspectV2.ALL_COLUMNS) + .where() + .eq(EbeanAspectV2.VERSION_COLUMN, ASPECT_LATEST_VERSION) + .eq(EbeanAspectV2.ASPECT_COLUMN, aspectName) + .like(EbeanAspectV2.URN_COLUMN, "urn:li:" + entityName + ":%"); + return exp.query().findStream().map(EbeanAspectV2::toEntityAspect); + } + @Override @Nonnull public Iterable listAllUrns(int start, int pageSize) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java index 02e36af343b071..5fdf4d45ffa3b3 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java @@ -318,7 +318,7 @@ public void removeEdgesFromNode( public void configure() { log.info("Setting up elastic graph index"); try { - for (ReindexConfig config : getReindexConfigs()) { + for (ReindexConfig config : buildReindexConfigs()) { _indexBuilder.buildIndex(config); } } catch (IOException e) { @@ -327,7 +327,7 @@ public void configure() { } @Override - public List getReindexConfigs() throws IOException { + public List buildReindexConfigs() throws IOException { return List.of(_indexBuilder.buildReindexState(_indexConvention.getIndexName(INDEX_NAME), GraphRelationshipMappingsBuilder.getMappings(), Collections.emptyMap())); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java index bf4dffe9e5fb8f..ef5a555e95ba89 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java @@ -46,8 +46,8 @@ public void configure() { } @Override - public List getReindexConfigs() { - return indexBuilders.getReindexConfigs(); + public List buildReindexConfigs() { + return indexBuilders.buildReindexConfigs(); } @Override diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java index 10c2fd725dca99..43431e93622f7d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java @@ -206,12 +206,7 @@ public void buildIndex(ReindexConfig indexState) throws IOException { // no need to reindex and only new mappings or dynamic settings // Just update the additional mappings - if (indexState.isPureMappingsAddition()) { - log.info("Updating index {} mappings in place.", indexState.name()); - PutMappingRequest request = new PutMappingRequest(indexState.name()).source(indexState.targetMappings()); - _searchClient.indices().putMapping(request, RequestOptions.DEFAULT); - log.info("Updated index {} with new mappings", indexState.name()); - } + applyMappings(indexState, true); if (indexState.requiresApplySettings()) { UpdateSettingsRequest request = new UpdateSettingsRequest(indexState.name()); @@ -234,6 +229,26 @@ public void buildIndex(ReindexConfig indexState) throws IOException { } } + /** + * Apply mappings changes if reindex is not required + * @param indexState the state of the current and target index settings/mappings + * @param suppressError during reindex logic this is not an error, for structured properties it is an error + * @throws IOException communication issues with ES + */ + public void applyMappings(ReindexConfig indexState, boolean suppressError) throws IOException { + if (indexState.isPureMappingsAddition()) { + log.info("Updating index {} mappings in place.", indexState.name()); + PutMappingRequest request = new PutMappingRequest(indexState.name()).source(indexState.targetMappings()); + _searchClient.indices().putMapping(request, RequestOptions.DEFAULT); + log.info("Updated index {} with new mappings", indexState.name()); + } else { + if (!suppressError) { + log.error("Attempted to apply invalid mappings. Current: {} Target: {}", indexState.currentMappings(), + indexState.targetMappings()); + } + } + } + public String reindexInPlaceAsync(String indexAlias, @Nullable QueryBuilder filterQuery, BatchWriteOperationsOptions options, ReindexConfig config) throws Exception { GetAliasesResponse aliasesResponse = _searchClient.indices().getAlias( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilder.java deleted file mode 100644 index 04c9f1993ff352..00000000000000 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilder.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.linkedin.metadata.search.elasticsearch.indexbuilder; - -import com.linkedin.metadata.models.EntitySpec; -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import com.linkedin.metadata.shared.ElasticSearchIndexed; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - - -@Slf4j -@RequiredArgsConstructor -public class EntityIndexBuilder implements ElasticSearchIndexed { - private final ESIndexBuilder indexBuilder; - private final EntitySpec entitySpec; - private final SettingsBuilder settingsBuilder; - private final String indexName; - - @Override - public void reindexAll() throws IOException { - log.info("Setting up index: {}", indexName); - for (ReindexConfig config : getReindexConfigs()) { - indexBuilder.buildIndex(config); - } - } - - @Override - public List getReindexConfigs() throws IOException { - Map mappings = MappingsBuilder.getMappings(entitySpec); - Map settings = settingsBuilder.getSettings(); - return List.of(indexBuilder.buildReindexState(indexName, mappings, settings)); - } -} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java index f38418058ca6d8..56cb26b09dc33c 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java @@ -3,8 +3,10 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; + import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; @@ -14,32 +16,37 @@ @RequiredArgsConstructor @Slf4j public class EntityIndexBuilders implements ElasticSearchIndexed { - private final ESIndexBuilder indexBuilder; - private final EntityRegistry entityRegistry; - private final IndexConvention indexConvention; - private final SettingsBuilder settingsBuilder; - - @Override - public void reindexAll() { - for (ReindexConfig config : getReindexConfigs()) { - try { - indexBuilder.buildIndex(config); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - - @Override - public List getReindexConfigs() { - return entityRegistry.getEntitySpecs().values().stream().flatMap(entitySpec -> { - try { - return new EntityIndexBuilder(indexBuilder, entitySpec, settingsBuilder, indexConvention.getIndexName(entitySpec)) - .getReindexConfigs().stream(); - } catch (IOException e) { + private final ESIndexBuilder indexBuilder; + private final EntityRegistry entityRegistry; + private final IndexConvention indexConvention; + private final SettingsBuilder settingsBuilder; + + public ESIndexBuilder getIndexBuilder() { + return indexBuilder; + } + + @Override + public void reindexAll() { + for (ReindexConfig config : buildReindexConfigs()) { + try { + indexBuilder.buildIndex(config); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public List buildReindexConfigs() { + Map settings = settingsBuilder.getSettings(); + return entityRegistry.getEntitySpecs().values().stream().map(entitySpec -> { + try { + Map mappings = MappingsBuilder.getMappings(entitySpec); + return indexBuilder.buildReindexState(indexConvention.getIndexName(entitySpec), mappings, settings); + } catch (IOException e) { throw new RuntimeException(e); - } } - ).collect(Collectors.toList()); - } + } + ).collect(Collectors.toList()); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java index b3e05d966e36b7..004b2e0a2adc4c 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java @@ -51,6 +51,8 @@ public static Map getPartialNgramConfigWithOverrides(Map getMappings(@Nonnull final EntitySpec entitySp mappings.put("urn", getMappingsForUrn()); mappings.put("runId", getMappingsForRunId()); - return ImmutableMap.of("properties", mappings); + return ImmutableMap.of(PROPERTIES, mappings); } private static Map getMappingsForUrn() { @@ -98,42 +100,9 @@ private static Map getMappingsForField(@Nonnull final Searchable Map mappings = new HashMap<>(); Map mappingForField = new HashMap<>(); if (fieldType == FieldType.KEYWORD) { - mappingForField.put(TYPE, KEYWORD); - mappingForField.put(NORMALIZER, KEYWORD_NORMALIZER); - // Add keyword subfield without lowercase filter - mappingForField.put(FIELDS, ImmutableMap.of(KEYWORD, KEYWORD_TYPE_MAP)); + mappingForField.putAll(getMappingsForKeyword()); } else if (fieldType == FieldType.TEXT || fieldType == FieldType.TEXT_PARTIAL || fieldType == FieldType.WORD_GRAM) { - mappingForField.put(TYPE, KEYWORD); - mappingForField.put(NORMALIZER, KEYWORD_NORMALIZER); - Map subFields = new HashMap<>(); - if (fieldType == FieldType.TEXT_PARTIAL || fieldType == FieldType.WORD_GRAM) { - subFields.put(NGRAM, getPartialNgramConfigWithOverrides( - ImmutableMap.of( - ANALYZER, PARTIAL_ANALYZER - ) - )); - if (fieldType == FieldType.WORD_GRAM) { - for (Map.Entry entry : Map.of( - WORD_GRAMS_LENGTH_2, WORD_GRAM_2_ANALYZER, - WORD_GRAMS_LENGTH_3, WORD_GRAM_3_ANALYZER, - WORD_GRAMS_LENGTH_4, WORD_GRAM_4_ANALYZER).entrySet()) { - String fieldName = entry.getKey(); - String analyzerName = entry.getValue(); - subFields.put(fieldName, ImmutableMap.of( - TYPE, TEXT, - ANALYZER, analyzerName - )); - } - } - } - subFields.put(DELIMITED, ImmutableMap.of( - TYPE, TEXT, - ANALYZER, TEXT_ANALYZER, - SEARCH_ANALYZER, TEXT_SEARCH_ANALYZER, - SEARCH_QUOTE_ANALYZER, CUSTOM_QUOTE_ANALYZER)); - // Add keyword subfield without lowercase filter - subFields.put(KEYWORD, KEYWORD_TYPE_MAP); - mappingForField.put(FIELDS, subFields); + mappingForField.putAll(getMappingsForSearchText(fieldType)); } else if (fieldType == FieldType.BROWSE_PATH) { mappingForField.put(TYPE, TEXT); mappingForField.put(FIELDS, @@ -189,6 +158,51 @@ private static Map getMappingsForField(@Nonnull final Searchable return mappings; } + private static Map getMappingsForKeyword() { + Map mappingForField = new HashMap<>(); + mappingForField.put(TYPE, KEYWORD); + mappingForField.put(NORMALIZER, KEYWORD_NORMALIZER); + // Add keyword subfield without lowercase filter + mappingForField.put(FIELDS, ImmutableMap.of(KEYWORD, KEYWORD_TYPE_MAP)); + return mappingForField; + } + + private static Map getMappingsForSearchText(FieldType fieldType) { + Map mappingForField = new HashMap<>(); + mappingForField.put(TYPE, KEYWORD); + mappingForField.put(NORMALIZER, KEYWORD_NORMALIZER); + Map subFields = new HashMap<>(); + if (fieldType == FieldType.TEXT_PARTIAL || fieldType == FieldType.WORD_GRAM) { + subFields.put(NGRAM, getPartialNgramConfigWithOverrides( + ImmutableMap.of( + ANALYZER, PARTIAL_ANALYZER + ) + )); + if (fieldType == FieldType.WORD_GRAM) { + for (Map.Entry entry : Map.of( + WORD_GRAMS_LENGTH_2, WORD_GRAM_2_ANALYZER, + WORD_GRAMS_LENGTH_3, WORD_GRAM_3_ANALYZER, + WORD_GRAMS_LENGTH_4, WORD_GRAM_4_ANALYZER).entrySet()) { + String fieldName = entry.getKey(); + String analyzerName = entry.getValue(); + subFields.put(fieldName, ImmutableMap.of( + TYPE, TEXT, + ANALYZER, analyzerName + )); + } + } + } + subFields.put(DELIMITED, ImmutableMap.of( + TYPE, TEXT, + ANALYZER, TEXT_ANALYZER, + SEARCH_ANALYZER, TEXT_SEARCH_ANALYZER, + SEARCH_QUOTE_ANALYZER, CUSTOM_QUOTE_ANALYZER)); + // Add keyword subfield without lowercase filter + subFields.put(KEYWORD, KEYWORD_TYPE_MAP); + mappingForField.put(FIELDS, subFields); + return mappingForField; + } + private static Map getMappingsForSearchScoreField( @Nonnull final SearchScoreFieldSpec searchScoreFieldSpec) { return ImmutableMap.of(searchScoreFieldSpec.getSearchScoreAnnotation().getFieldName(), diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java index 4f5f2926d3da03..8b8a48f5d9cdac 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java @@ -121,13 +121,14 @@ public ReindexConfig build() { if (super.exists) { /* Consider mapping changes */ MapDifference mappingsDiff = Maps.difference( - (TreeMap) super.currentMappings.getOrDefault("properties", new TreeMap()), - (TreeMap) super.targetMappings.getOrDefault("properties", new TreeMap())); + getOrDefault(super.currentMappings, List.of("properties")), + getOrDefault(super.targetMappings, List.of("properties"))); super.requiresApplyMappings = !mappingsDiff.entriesDiffering().isEmpty() || !mappingsDiff.entriesOnlyOnRight().isEmpty(); super.isPureMappingsAddition = super.requiresApplyMappings && mappingsDiff.entriesDiffering().isEmpty() && !mappingsDiff.entriesOnlyOnRight().isEmpty(); + if (super.requiresApplyMappings && super.isPureMappingsAddition) { log.info("Index: {} - New fields have been added to index. Adding: {}", super.name, mappingsDiff.entriesOnlyOnRight()); @@ -171,8 +172,21 @@ public ReindexConfig build() { return super.build(); } + private static TreeMap getOrDefault(Map map, List path) { + if (map == null) { + return new TreeMap<>(); + } + + TreeMap item = (TreeMap) map.getOrDefault(path.get(0), new TreeMap()); + if (path.size() == 1) { + return item; + } else { + return getOrDefault(item, path.subList(1, path.size())); + } + } + private boolean isAnalysisEqual() { - if (!super.targetSettings.containsKey("index")) { + if (super.targetSettings == null || !super.targetSettings.containsKey("index")) { return true; } Map indexSettings = (Map) super.targetSettings.get("index"); @@ -186,7 +200,7 @@ private boolean isAnalysisEqual() { } private boolean isSettingsEqual() { - if (!super.targetSettings.containsKey("index")) { + if (super.targetSettings == null || !super.targetSettings.containsKey("index")) { return true; } Map indexSettings = (Map) super.targetSettings.get("index"); @@ -196,7 +210,7 @@ private boolean isSettingsEqual() { } private boolean isSettingsReindexRequired() { - if (!super.targetSettings.containsKey("index")) { + if (super.targetSettings == null || !super.targetSettings.containsKey("index")) { return false; } Map indexSettings = (Map) super.targetSettings.get("index"); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java index 76f4736f2746e2..49809cf9339367 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java @@ -7,6 +7,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.data.schema.DataSchema; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.SearchScoreFieldSpec; @@ -21,6 +22,7 @@ import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import javax.annotation.Nonnull; @@ -30,6 +32,7 @@ * Class that provides a utility function that transforms the snapshot object into a search document */ @Slf4j +@Setter @RequiredArgsConstructor public class SearchDocumentTransformer { @@ -42,6 +45,8 @@ public class SearchDocumentTransformer { // Maximum customProperties value length private final int maxValueLength; + private SystemEntityClient entityClient; + private static final String BROWSE_PATH_V2_DELIMITER = "␟"; public Optional transformSnapshot(final RecordTemplate snapshot, final EntitySpec entitySpec, @@ -72,14 +77,18 @@ public Optional transformAspect( FieldExtractor.extractFields(aspect, aspectSpec.getSearchableFieldSpecs(), maxValueLength); final Map> extractedSearchScoreFields = FieldExtractor.extractFields(aspect, aspectSpec.getSearchScoreFieldSpecs(), maxValueLength); - if (extractedSearchableFields.isEmpty() && extractedSearchScoreFields.isEmpty()) { - return Optional.empty(); + + Optional result = Optional.empty(); + + if (!extractedSearchableFields.isEmpty() || !extractedSearchScoreFields.isEmpty()) { + final ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); + searchDocument.put("urn", urn.toString()); + extractedSearchableFields.forEach((key, values) -> setSearchableValue(key, values, searchDocument, forDelete)); + extractedSearchScoreFields.forEach((key, values) -> setSearchScoreValue(key, values, searchDocument, forDelete)); + result = Optional.of(searchDocument.toString()); } - final ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); - searchDocument.put("urn", urn.toString()); - extractedSearchableFields.forEach((key, values) -> setSearchableValue(key, values, searchDocument, forDelete)); - extractedSearchScoreFields.forEach((key, values) -> setSearchScoreValue(key, values, searchDocument, forDelete)); - return Optional.of(searchDocument.toString()); + + return result; } public void setSearchableValue(final SearchableFieldSpec fieldSpec, final List fieldValues, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java index 36b685f084d51c..ea7286112f870f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java @@ -12,6 +12,7 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.dataset.FineGrainedLineage; import com.linkedin.dataset.UpstreamLineage; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.graph.Edge; @@ -28,6 +29,7 @@ import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; import com.linkedin.metadata.search.utils.SearchUtils; import com.linkedin.metadata.systemmetadata.SystemMetadataService; @@ -39,6 +41,8 @@ import com.linkedin.mxe.MetadataChangeLog; import com.linkedin.mxe.SystemMetadata; import com.linkedin.util.Pair; + +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.ArrayList; @@ -68,6 +72,7 @@ public class UpdateIndicesService { private final SystemMetadataService _systemMetadataService; private final EntityRegistry _entityRegistry; private final SearchDocumentTransformer _searchDocumentTransformer; + private final EntityIndexBuilders _entityIndexBuilders; @Value("${featureFlags.graphServiceDiffModeEnabled:true}") private boolean _graphDiffMode; @@ -90,25 +95,31 @@ public void setSearchDiffMode(boolean searchDiffMode) { } public UpdateIndicesService( - GraphService graphService, - EntitySearchService entitySearchService, - TimeseriesAspectService timeseriesAspectService, - SystemMetadataService systemMetadataService, - EntityRegistry entityRegistry, - SearchDocumentTransformer searchDocumentTransformer) { + GraphService graphService, + EntitySearchService entitySearchService, + TimeseriesAspectService timeseriesAspectService, + SystemMetadataService systemMetadataService, + EntityRegistry entityRegistry, + SearchDocumentTransformer searchDocumentTransformer, + EntityIndexBuilders entityIndexBuilders) { _graphService = graphService; _entitySearchService = entitySearchService; _timeseriesAspectService = timeseriesAspectService; _systemMetadataService = systemMetadataService; _entityRegistry = entityRegistry; _searchDocumentTransformer = searchDocumentTransformer; + _entityIndexBuilders = entityIndexBuilders; } public void handleChangeEvent(@Nonnull final MetadataChangeLog event) { - if (UPDATE_CHANGE_TYPES.contains(event.getChangeType())) { - handleUpdateChangeEvent(event); - } else if (event.getChangeType() == ChangeType.DELETE) { - handleDeleteChangeEvent(event); + try { + if (UPDATE_CHANGE_TYPES.contains(event.getChangeType())) { + handleUpdateChangeEvent(event); + } else if (event.getChangeType() == ChangeType.DELETE) { + handleDeleteChangeEvent(event); + } + } catch (IOException e) { + throw new RuntimeException(e); } } @@ -123,7 +134,7 @@ public void handleChangeEvent(@Nonnull final MetadataChangeLog event) { * * @param event the change event to be processed. */ - public void handleUpdateChangeEvent(@Nonnull final MetadataChangeLog event) { + public void handleUpdateChangeEvent(@Nonnull final MetadataChangeLog event) throws IOException { final EntitySpec entitySpec = getEventEntitySpec(event); final Urn urn = EntityKeyUtils.getUrnFromLog(event, entitySpec.getKeyAspectSpec()); @@ -212,7 +223,7 @@ public void handleDeleteChangeEvent(@Nonnull final MetadataChangeLog event) { if (!aspectSpec.isTimeseries()) { deleteSystemMetadata(urn, aspectSpec, isDeletingKey); deleteGraphData(urn, aspectSpec, aspect, isDeletingKey, event); - deleteSearchData(urn, entitySpec.getName(), aspectSpec, aspect, isDeletingKey); + deleteSearchData(_entitySearchService, urn, entitySpec.getName(), aspectSpec, aspect, isDeletingKey); } } @@ -405,7 +416,8 @@ private static List getMergedEdges(final Set oldEdgeSet, final Set searchDocument; Optional previousSearchDocument = Optional.empty(); @@ -513,7 +525,8 @@ private void deleteGraphData( } } - private void deleteSearchData(Urn urn, String entityName, AspectSpec aspectSpec, RecordTemplate aspect, Boolean isKeyAspect) { + private void deleteSearchData(EntitySearchService entitySearchService, Urn urn, String entityName, + AspectSpec aspectSpec, RecordTemplate aspect, Boolean isKeyAspect) { String docId; try { docId = URLEncoder.encode(urn.toString(), "UTF-8"); @@ -551,4 +564,13 @@ private EntitySpec getEventEntitySpec(@Nonnull final MetadataChangeLog event) { event.getEntityType())); } } + + /** + * Allow internal use of the system entity client. Solves recursive dependencies between the UpdateIndicesService + * and the SystemJavaEntityClient + * @param systemEntityClient system entity client + */ + public void setSystemEntityClient(SystemEntityClient systemEntityClient) { + _searchDocumentTransformer.setEntityClient(systemEntityClient); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/shared/ElasticSearchIndexed.java b/metadata-io/src/main/java/com/linkedin/metadata/shared/ElasticSearchIndexed.java index 1f13cb8321284d..64ad88c08a7410 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/shared/ElasticSearchIndexed.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/shared/ElasticSearchIndexed.java @@ -11,7 +11,7 @@ public interface ElasticSearchIndexed { * The index configurations for the given service. * @return List of reindex configurations */ - List getReindexConfigs() throws IOException; + List buildReindexConfigs() throws IOException; /** * Mirrors the service's functions which diff --git a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java index dd8e19861ccd21..e9ee1d6ee78d5d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java @@ -205,7 +205,7 @@ public List listRuns(Integer pageOffset, Integer pageSize, public void configure() { log.info("Setting up system metadata index"); try { - for (ReindexConfig config : getReindexConfigs()) { + for (ReindexConfig config : buildReindexConfigs()) { _indexBuilder.buildIndex(config); } } catch (IOException ie) { @@ -214,7 +214,7 @@ public void configure() { } @Override - public List getReindexConfigs() throws IOException { + public List buildReindexConfigs() throws IOException { return List.of(_indexBuilder.buildReindexState(_indexConvention.getIndexName(INDEX_NAME), SystemMetadataMappingsBuilder.getMappings(), Collections.emptyMap())); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java index 43ba87f474d6aa..a496fc427138e9 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java @@ -137,9 +137,10 @@ public void configure() { } @Override - public List getReindexConfigs() { - return _indexBuilders.getReindexConfigs(); + public List buildReindexConfigs() { + return _indexBuilders.buildReindexConfigs(); } + public String reindexAsync(String index, @Nullable QueryBuilder filterQuery, BatchWriteOperationsOptions options) throws Exception { return _indexBuilders.reindexAsync(index, filterQuery, options); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java index b0751a9c6f9ea5..e9518ed8c39fa0 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java @@ -29,7 +29,7 @@ public class TimeseriesAspectIndexBuilders implements ElasticSearchIndexed { @Override public void reindexAll() { - for (ReindexConfig config : getReindexConfigs()) { + for (ReindexConfig config : buildReindexConfigs()) { try { _indexBuilder.buildIndex(config); } catch (IOException e) { @@ -63,7 +63,7 @@ public String reindexAsync(String index, @Nullable QueryBuilder filterQuery, Bat } @Override - public List getReindexConfigs() { + public List buildReindexConfigs() { return _entityRegistry.getEntitySpecs().values().stream() .flatMap(entitySpec -> entitySpec.getAspectSpecs().stream() .map(aspectSpec -> Pair.of(entitySpec, aspectSpec))) @@ -80,4 +80,5 @@ public List getReindexConfigs() { } }).collect(Collectors.toList()); } + } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java index 38b2ed4ed199a0..30d821662d3779 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanAspectMigrationsDaoTest.java @@ -1,18 +1,27 @@ package com.linkedin.metadata.entity; +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.AspectIngestionUtils; import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.EbeanTestUtils; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.entity.ebean.EbeanRetentionService; import com.linkedin.metadata.event.EventProducer; +import com.linkedin.metadata.key.CorpUserKey; import com.linkedin.metadata.models.registry.EntityRegistryException; import com.linkedin.metadata.service.UpdateIndicesService; import io.ebean.Database; -import org.testng.Assert; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import static com.linkedin.metadata.Constants.*; import static org.mockito.Mockito.*; +import static org.testng.Assert.*; public class EbeanAspectMigrationsDaoTest extends AspectMigrationsDaoTest { @@ -37,13 +46,19 @@ public void setupTest() { _migrationsDao = dao; } - /** - * Ideally, all tests would be in the base class, so they're reused between all implementations. - * When that's the case - test runner will ignore this class (and its base!) so we keep this dummy test - * to make sure this class will always be discovered. - */ @Test - public void obligatoryTest() throws AssertionError { - Assert.assertTrue(true); + public void testStreamAspects() throws AssertionError { + final int totalAspects = 30; + Map ingestedAspects = + AspectIngestionUtils.ingestCorpUserKeyAspects(_entityServiceImpl, totalAspects); + List ingestedUrns = ingestedAspects.keySet().stream().map(Urn::toString).collect(Collectors.toList()); + + Stream aspectStream = _migrationsDao.streamAspects(CORP_USER_ENTITY_NAME, CORP_USER_KEY_ASPECT_NAME); + List aspectList = aspectStream.collect(Collectors.toList()); + assertEquals(ingestedUrns.size(), aspectList.size()); + Set urnsFetched = aspectList.stream().map(EntityAspect::getUrn).collect(Collectors.toSet()); + for (String urn : ingestedUrns) { + assertTrue(urnsFetched.contains(urn)); + } } } diff --git a/metadata-io/src/test/java/io/datahubproject/test/DataGenerator.java b/metadata-io/src/test/java/io/datahubproject/test/DataGenerator.java index cfa9c1258583d7..12a02f954e1bc2 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/DataGenerator.java +++ b/metadata-io/src/test/java/io/datahubproject/test/DataGenerator.java @@ -12,11 +12,16 @@ import com.linkedin.events.metadata.ChangeType; import com.linkedin.glossary.GlossaryTermInfo; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.config.PreProcessHooks; +import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.AspectUtils; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityServiceImpl; +import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import net.datafaker.Faker; @@ -42,6 +47,8 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.mockito.Mockito.mock; + public class DataGenerator { private final static Faker FAKER = new Faker(); private final EntityRegistry entityRegistry; @@ -52,10 +59,21 @@ public DataGenerator(EntityService entityService) { this.entityRegistry = entityService.getEntityRegistry(); } + public static DataGenerator build(EntityRegistry entityRegistry) { + EntityServiceImpl mockEntityServiceImpl = new EntityServiceImpl(mock(AspectDao.class), + mock(EventProducer.class), entityRegistry, false, + mock(UpdateIndicesService.class), mock(PreProcessHooks.class)); + return new DataGenerator(mockEntityServiceImpl); + } + public Stream> generateDatasets() { return generateMCPs("dataset", 10, List.of()); } + public List generateTags(long count) { + return generateMCPs("tag", count, List.of()).findFirst().get(); + } + public Stream> generateMCPs(String entityName, long count, List aspects) { EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); @@ -127,9 +145,7 @@ public Stream> generateMCPs(String entityName, long public Map>> nestedRandomAspectGenerators = Map.of( "globalTags", (aspect, count) -> { try { - List tags = generateMCPs("tag", count, List.of()) - .map(mcps -> mcps.get(0)) - .collect(Collectors.toList()); + List tags = generateTags(count); Method setTagsMethod = aspect.getClass().getMethod("setTags", TagAssociationArray.class); TagAssociationArray tagAssociations = new TagAssociationArray(); tagAssociations.addAll(tags.stream().map( diff --git a/metadata-jobs/mae-consumer-job/src/main/resources/application.properties b/metadata-jobs/mae-consumer-job/src/main/resources/application.properties index 6befa3e8789d88..7df61c93ab66d4 100644 --- a/metadata-jobs/mae-consumer-job/src/main/resources/application.properties +++ b/metadata-jobs/mae-consumer-job/src/main/resources/application.properties @@ -3,4 +3,4 @@ management.endpoints.web.exposure.include=metrics, health, info spring.mvc.servlet.path=/ management.health.elasticsearch.enabled=false management.health.neo4j.enabled=false - +entityClient.preferredImpl=restli diff --git a/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTestConfiguration.java b/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTestConfiguration.java index a214117f4e1bcb..aa097a52c8fc64 100644 --- a/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTestConfiguration.java +++ b/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTestConfiguration.java @@ -7,6 +7,7 @@ import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.models.registry.ConfigEntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; import com.linkedin.metadata.systemmetadata.ElasticSearchSystemMetadataService; import io.ebean.Database; import org.springframework.boot.test.context.TestConfiguration; @@ -40,4 +41,7 @@ public class MaeConsumerApplicationTestConfiguration { @MockBean private ConfigEntityRegistry _configEntityRegistry; + + @MockBean + public EntityIndexBuilders entityIndexBuilders; } diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeLogProcessor.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeLogProcessor.java index 64f89c595163da..796f570a1732ec 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeLogProcessor.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeLogProcessor.java @@ -14,6 +14,8 @@ import com.linkedin.metadata.utils.metrics.MetricUtils; import com.linkedin.mxe.MetadataChangeLog; import com.linkedin.mxe.Topics; + +import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; import lombok.Getter; @@ -47,7 +49,10 @@ public class MetadataChangeLogProcessor { @Autowired public MetadataChangeLogProcessor(List metadataChangeLogHooks) { - this.hooks = metadataChangeLogHooks.stream().filter(MetadataChangeLogHook::isEnabled).collect(Collectors.toList()); + this.hooks = metadataChangeLogHooks.stream() + .filter(MetadataChangeLogHook::isEnabled) + .sorted(Comparator.comparing(MetadataChangeLogHook::executionOrder)) + .collect(Collectors.toList()); this.hooks.forEach(MetadataChangeLogHook::init); } diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/MetadataChangeLogHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/MetadataChangeLogHook.java index c7857eb7baffc1..39b47768a6dcf0 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/MetadataChangeLogHook.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/MetadataChangeLogHook.java @@ -29,4 +29,12 @@ default boolean isEnabled() { * Invoke the hook when a MetadataChangeLog is received */ void invoke(@Nonnull MetadataChangeLog log) throws Exception; + + /** + * Controls hook execution ordering + * @return order to execute + */ + default int executionOrder() { + return 100; + } } diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java index fad7a340749644..78c87ec8f4b3b8 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHook.java @@ -24,7 +24,7 @@ EntityRegistryFactory.class, SystemMetadataServiceFactory.class, SearchDocumentTransformerFactory.class}) public class UpdateIndicesHook implements MetadataChangeLogHook { - private final UpdateIndicesService _updateIndicesService; + protected final UpdateIndicesService _updateIndicesService; private final boolean _isEnabled; public UpdateIndicesHook( diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java index 030ca831314330..90f8f208c4cb6f 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java @@ -34,6 +34,7 @@ import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.systemmetadata.SystemMetadataService; @@ -42,10 +43,12 @@ import com.linkedin.mxe.MetadataChangeLog; import com.linkedin.mxe.SystemMetadata; import com.linkedin.schema.SchemaField; + import java.net.URISyntaxException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Value; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -82,9 +85,13 @@ public class UpdateIndicesHookTest { private SearchDocumentTransformer _searchDocumentTransformer; private DataHubUpgradeKafkaListener _mockDataHubUpgradeKafkaListener; private ConfigurationProvider _mockConfigurationProvider; + private EntityIndexBuilders _mockEntityIndexBuilders; private Urn _actorUrn; private UpdateIndicesService _updateIndicesService; + @Value("${elasticsearch.index.maxArrayLength}") + private int maxArrayLength; + @BeforeMethod public void setupTest() { _actorUrn = UrnUtils.getUrn(TEST_ACTOR_URN); @@ -95,6 +102,8 @@ public void setupTest() { _searchDocumentTransformer = new SearchDocumentTransformer(1000, 1000, 1000); _mockDataHubUpgradeKafkaListener = Mockito.mock(DataHubUpgradeKafkaListener.class); _mockConfigurationProvider = Mockito.mock(ConfigurationProvider.class); + _mockEntityIndexBuilders = Mockito.mock(EntityIndexBuilders.class); + ElasticSearchConfiguration elasticSearchConfiguration = new ElasticSearchConfiguration(); SystemUpdateConfiguration systemUpdateConfiguration = new SystemUpdateConfiguration(); systemUpdateConfiguration.setWaitForSystemUpdate(false); @@ -105,7 +114,8 @@ public void setupTest() { _mockTimeseriesAspectService, _mockSystemMetadataService, ENTITY_REGISTRY, - _searchDocumentTransformer + _searchDocumentTransformer, + _mockEntityIndexBuilders ); _updateIndicesHook = new UpdateIndicesHook( _updateIndicesService, @@ -163,7 +173,8 @@ public void testInputFieldsEdgesAreAdded() throws Exception { _mockTimeseriesAspectService, _mockSystemMetadataService, mockEntityRegistry, - _searchDocumentTransformer + _searchDocumentTransformer, + _mockEntityIndexBuilders ); _updateIndicesHook = new UpdateIndicesHook(_updateIndicesService, true); diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java index dc5a6cd23295b5..1d9c17c6769908 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java @@ -9,6 +9,7 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.registry.SchemaRegistryService; import com.linkedin.metadata.search.elasticsearch.ElasticSearchService; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; import com.linkedin.metadata.systemmetadata.SystemMetadataService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; @@ -64,4 +65,7 @@ public class MCLSpringTestConfiguration { @MockBean public SchemaRegistryService schemaRegistryService; + + @MockBean + public EntityIndexBuilders entityIndexBuilders; } diff --git a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java index 558a7b9d90ccbd..bee1441b5aaf63 100644 --- a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java +++ b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java @@ -8,6 +8,7 @@ import com.linkedin.metadata.models.registry.ConfigEntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.restli.DefaultRestliClientFactory; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.parseq.retry.backoff.ExponentialBackoff; import com.linkedin.restli.client.Client; @@ -57,4 +58,7 @@ public RestliEntityClient restliEntityClient() { @MockBean protected SiblingGraphService siblingGraphService; + + @MockBean + public EntityIndexBuilders entityIndexBuilders; } diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml index 42749d8205d215..f180a3f42b7308 100644 --- a/metadata-service/configuration/src/main/resources/application.yml +++ b/metadata-service/configuration/src/main/resources/application.yml @@ -339,7 +339,7 @@ cache: statsEnabled: ${CACHE_CLIENT_ENTITY_CLIENT_STATS_ENABLED:true} statsIntervalSeconds: ${CACHE_CLIENT_ENTITY_CLIENT_STATS_INTERVAL_SECONDS:120} defaultTTLSeconds: ${CACHE_CLIENT_ENTITY_CLIENT_TTL_SECONDS:0} # do not cache entity/aspects by default - maxBytes: ${CACHE_CLIENT_USAGE_ENTITY_MAX_BYTES:104857600} # 100MB + maxBytes: ${CACHE_CLIENT_ENTITY_CLIENT_MAX_BYTES:104857600} # 100MB entityAspectTTLSeconds: # cache user aspects for 20s corpuser: diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java index 5122be69982f0f..f1c1a7b743714a 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java @@ -33,17 +33,19 @@ public class EntityServiceFactory { TopicConventionFactory.TOPIC_CONVENTION_BEAN, "entityRegistry"}) @Nonnull protected EntityService createInstance( - Producer producer, - TopicConvention convention, - KafkaHealthChecker kafkaHealthChecker, - @Qualifier("entityAspectDao") AspectDao aspectDao, - EntityRegistry entityRegistry, - ConfigurationProvider configurationProvider, - UpdateIndicesService updateIndicesService) { + Producer producer, + TopicConvention convention, + KafkaHealthChecker kafkaHealthChecker, + @Qualifier("entityAspectDao") AspectDao aspectDao, + EntityRegistry entityRegistry, + ConfigurationProvider configurationProvider, + UpdateIndicesService updateIndicesService) { final KafkaEventProducer eventProducer = new KafkaEventProducer(producer, convention, kafkaHealthChecker); FeatureFlags featureFlags = configurationProvider.getFeatureFlags(); - return new EntityServiceImpl(aspectDao, eventProducer, entityRegistry, + EntityService entityService = new EntityServiceImpl(aspectDao, eventProducer, entityRegistry, featureFlags.isAlwaysEmitChangeLog(), updateIndicesService, featureFlags.getPreProcessHooks(), _ebeanMaxTransactionRetry); + + return entityService; } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java index e1c24b805437b6..3f2388f4829e31 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java @@ -16,14 +16,17 @@ import com.linkedin.metadata.timeseries.TimeseriesAspectService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; @Configuration +@ConditionalOnExpression("'${entityClient.preferredImpl:java}'.equals('java')") @Import({DataHubKafkaProducerFactory.class}) public class JavaEntityClientFactory { + @Autowired @Qualifier("entityService") private EntityService _entityService; @@ -74,7 +77,7 @@ public JavaEntityClient getJavaEntityClient(@Qualifier("restliEntityClient") fin public SystemJavaEntityClient systemJavaEntityClient(@Qualifier("configurationProvider") final ConfigurationProvider configurationProvider, @Qualifier("systemAuthentication") final Authentication systemAuthentication, @Qualifier("systemRestliEntityClient") final RestliEntityClient restliEntityClient) { - return new SystemJavaEntityClient( + SystemJavaEntityClient systemJavaEntityClient = new SystemJavaEntityClient( _entityService, _deleteEntityService, _entitySearchService, @@ -86,5 +89,9 @@ public SystemJavaEntityClient systemJavaEntityClient(@Qualifier("configurationPr restliEntityClient, systemAuthentication, configurationProvider.getCache().getClient().getEntityClient()); + + _entityService.setSystemEntityClient(systemJavaEntityClient); + + return systemJavaEntityClient; } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/update/indices/UpdateIndicesServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/update/indices/UpdateIndicesServiceFactory.java index f86f6bf7d08777..a4ea02af94bad3 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/update/indices/UpdateIndicesServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/update/indices/UpdateIndicesServiceFactory.java @@ -1,24 +1,44 @@ package com.linkedin.gms.factory.entity.update.indices; +import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.gms.factory.search.EntityIndexBuildersFactory; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.systemmetadata.SystemMetadataService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; @Configuration +@Import(EntityIndexBuildersFactory.class) public class UpdateIndicesServiceFactory { + @Autowired + private ApplicationContext context; + @Value("${entityClient.preferredImpl:java}") + private String entityClientImpl; @Bean public UpdateIndicesService updateIndicesService(GraphService graphService, EntitySearchService entitySearchService, - TimeseriesAspectService timeseriesAspectService, SystemMetadataService systemMetadataService, - EntityRegistry entityRegistry, SearchDocumentTransformer searchDocumentTransformer) { - return new UpdateIndicesService(graphService, entitySearchService, timeseriesAspectService, - systemMetadataService, entityRegistry, searchDocumentTransformer); + TimeseriesAspectService timeseriesAspectService, + SystemMetadataService systemMetadataService, + EntityRegistry entityRegistry, SearchDocumentTransformer searchDocumentTransformer, + EntityIndexBuilders entityIndexBuilders) { + UpdateIndicesService updateIndicesService = new UpdateIndicesService(graphService, entitySearchService, timeseriesAspectService, + systemMetadataService, entityRegistry, searchDocumentTransformer, entityIndexBuilders); + + if ("restli".equals(entityClientImpl)) { + updateIndicesService.setSystemEntityClient(context.getBean(SystemRestliEntityClient.class)); + } + + return updateIndicesService; } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java index a2a0dbaf89c79c..6d8a62ac1fd187 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java @@ -47,6 +47,9 @@ public class ElasticSearchServiceFactory { @Qualifier("settingsBuilder") private SettingsBuilder settingsBuilder; + @Autowired + private EntityIndexBuilders entityIndexBuilders; + @Autowired private ConfigurationProvider configurationProvider; @@ -64,9 +67,7 @@ protected ElasticSearchService getInstance(ConfigurationProvider configurationPr new ESSearchDAO(entityRegistry, components.getSearchClient(), components.getIndexConvention(), configurationProvider.getFeatureFlags().isPointInTimeCreationEnabled(), elasticSearchConfiguration.getImplementation(), searchConfiguration, customSearchConfiguration); - return new ElasticSearchService( - new EntityIndexBuilders(components.getIndexBuilder(), entityRegistry, components.getIndexConvention(), - settingsBuilder), esSearchDAO, + return new ElasticSearchService(entityIndexBuilders, esSearchDAO, new ESBrowseDAO(entityRegistry, components.getSearchClient(), components.getIndexConvention(), searchConfiguration, customSearchConfiguration), new ESWriteDAO(entityRegistry, components.getSearchClient(), components.getIndexConvention(), diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/EntityIndexBuildersFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/EntityIndexBuildersFactory.java new file mode 100644 index 00000000000000..6bb206ee3ad61f --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/EntityIndexBuildersFactory.java @@ -0,0 +1,35 @@ +package com.linkedin.gms.factory.search; + +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder; +import com.linkedin.metadata.spring.YamlPropertySourceFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; + + +@Configuration +@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +public class EntityIndexBuildersFactory { + + @Autowired + @Qualifier("baseElasticSearchComponents") + private BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components; + + @Autowired + @Qualifier("entityRegistry") + private EntityRegistry entityRegistry; + + @Autowired + @Qualifier("settingsBuilder") + private SettingsBuilder settingsBuilder; + + + @Bean + protected EntityIndexBuilders entityIndexBuilders() { + return new EntityIndexBuilders(components.getIndexBuilder(), entityRegistry, components.getIndexConvention(), settingsBuilder); + } +} \ No newline at end of file diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClientCache.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClientCache.java index 3b35dc528915ab..6006f3a9a87f64 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClientCache.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClientCache.java @@ -21,7 +21,6 @@ import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import java.util.stream.StreamSupport; import static com.linkedin.metadata.utils.PegasusUtils.urnToEntityName; @@ -44,8 +43,7 @@ public Map batchGetV2(@Nonnull final Set urns, @Nonnul if (config.isEnabled()) { Set keys = urns.stream() - .flatMap(urn -> aspectNames.stream() - .map(a -> Key.builder().urn(urn).aspectName(a).build())) + .flatMap(urn -> aspectNames.stream().map(a -> Key.builder().urn(urn).aspectName(a).build())) .collect(Collectors.toSet()); Map envelopedAspects = cache.getAll(keys); @@ -92,13 +90,13 @@ public EntityClientCache build(Class metricClazz) { Map> keysByEntity = StreamSupport.stream(keys.spliterator(), true) .collect(Collectors.groupingBy(Key::getEntityName, Collectors.toSet())); - Stream> results = keysByEntity.entrySet().parallelStream() + Map results = keysByEntity.entrySet().parallelStream() .flatMap(entry -> { Set urns = entry.getValue().stream() .map(Key::getUrn) .collect(Collectors.toSet()); Set aspects = entry.getValue().stream() - .map(Key::getEntityName) + .map(Key::getAspectName) .collect(Collectors.toSet()); return loadFunction.apply(urns, aspects).entrySet().stream(); }) @@ -106,9 +104,9 @@ public EntityClientCache build(Class metricClazz) { .map(envAspect -> { Key key = Key.builder().urn(resp.getKey()).aspectName(envAspect.getName()).build(); return Map.entry(key, envAspect); - })); + })).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - return results.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return results; }; // ideally the cache time comes from caching headers from service, but configuration driven for now diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java index 30cfc2e0288bdc..b7607053df8e30 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -9,6 +9,7 @@ import com.linkedin.entity.Entity; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; @@ -297,4 +298,11 @@ Pair>> generateDefaultAspectsOnFirstW */ @Nonnull BrowsePathsV2 buildDefaultBrowsePathV2(final @Nonnull Urn urn, boolean useContainerPaths) throws URISyntaxException; + + /** + * Allow internal use of the system entity client. Solves recursive dependencies between the EntityService + * and the SystemJavaEntityClient + * @param systemEntityClient system entity client + */ + void setSystemEntityClient(SystemEntityClient systemEntityClient); }