From 2e8ea464c7a96d4ec687b48e7fbc0bdcc987ec95 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Tue, 19 Nov 2024 10:31:00 -0600 Subject: [PATCH 1/5] Add metadata transformations with metadata tuples Signed-off-by: Andre Kurait --- MetadataMigration/build.gradle | 1 + .../migrations/MigrateOrEvaluateArgs.java | 37 ++++ .../migrations/commands/Evaluate.java | 6 +- .../migrations/commands/Migrate.java | 6 +- .../commands/MigratorEvaluatorBase.java | 24 ++- .../src/main/resources/log4j2.properties | 20 ++- .../bulkload/models/ComponentTemplate.java | 13 ++ .../bulkload/models/GlobalMetadata.java | 37 ++-- .../migrations/bulkload/models/Index.java | 13 ++ .../bulkload/models/IndexMetadata.java | 28 +-- .../bulkload/models/IndexTemplate.java | 13 ++ .../bulkload/models/LegacyTemplate.java | 13 ++ .../bulkload/models/MigrationItem.java | 27 +++ .../transformers/CompositeTransformer.java | 28 +++ .../TransformerToIJsonTransformerAdapter.java | 165 ++++++++++++++++++ .../GlobalMetadataData_ES_6_8.java | 14 +- .../IndexMetadataData_ES_6_8.java | 12 +- .../GlobalMetadataData_ES_7_10.java | 22 +-- .../IndexMetadataData_ES_7_10.java | 12 +- .../GlobalMetadataData_OS_2_11.java | 26 ++- .../IndexMetadataData_OS_2_11.java | 12 +- .../RemoteIndexMetadata.java | 51 +++--- .../version_universal/RemoteMetadata.java | 32 +--- .../bulkload/worker/IndexRunner.java | 21 ++- ...sformerForDocumentTypeRemovalProvider.java | 4 +- 25 files changed, 524 insertions(+), 113 deletions(-) create mode 100644 RFS/src/main/java/org/opensearch/migrations/bulkload/models/ComponentTemplate.java create mode 100644 RFS/src/main/java/org/opensearch/migrations/bulkload/models/Index.java create mode 100644 RFS/src/main/java/org/opensearch/migrations/bulkload/models/IndexTemplate.java create mode 100644 RFS/src/main/java/org/opensearch/migrations/bulkload/models/LegacyTemplate.java create mode 100644 RFS/src/main/java/org/opensearch/migrations/bulkload/models/MigrationItem.java create mode 100644 RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/CompositeTransformer.java create mode 100644 RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformerToIJsonTransformerAdapter.java diff --git a/MetadataMigration/build.gradle b/MetadataMigration/build.gradle index fa0bf72d9..dc66ccb5a 100644 --- a/MetadataMigration/build.gradle +++ b/MetadataMigration/build.gradle @@ -11,6 +11,7 @@ dependencies { implementation project(":coreUtilities") implementation project(":RFS") implementation project(':transformation') + implementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerLoaders') implementation group: 'org.jcommander', name: 'jcommander' implementation group: 'org.slf4j', name: 'slf4j-api' diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java b/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java index 01942be40..4c51a1868 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java @@ -4,9 +4,11 @@ import org.opensearch.migrations.bulkload.common.http.ConnectionContext; import org.opensearch.migrations.bulkload.models.DataFilterArgs; +import org.opensearch.migrations.transform.TransformerParams; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParametersDelegate; +import lombok.Getter; public class MigrateOrEvaluateArgs { @Parameter(names = {"--help", "-h"}, help = true, description = "Displays information about how to use this tool") @@ -52,4 +54,39 @@ public class MigrateOrEvaluateArgs { @Parameter(names = {"--source-version" }, description = "Version of the source cluster, for example: Elasticsearch 7.10 or OS 1.3.", converter = VersionConverter.class) public Version sourceVersion = null; + + @ParametersDelegate + public MetadataTransformerParams metadataTransformationParams = new MetadataTransformerParams(); + + @Getter + public static class MetadataTransformerParams implements TransformerParams { + public String getTransformerConfigParameterArgPrefix() { + return ""; + } + @Parameter( + required = false, + names = "--transformer-config-base64", + arity = 1, + description = "Configuration of metadata transformers. The same contents as --transformer-config but " + + "Base64 encoded so that the configuration is easier to pass as a command line parameter.") + private String transformerConfigEncoded; + + @Parameter( + required = false, + names = "--transformer-config", + arity = 1, + description = "Configuration of metadata transformers. Either as a string that identifies the " + + "transformer that should be run (with default settings) or as json to specify options " + + "as well as multiple transformers to run in sequence. " + + "For json, keys are the (simple) names of the loaded transformers and values are the " + + "configuration passed to each of the transformers.") + private String transformerConfig; + + @Parameter( + required = false, + names = "--transformer-config-file", + arity = 1, + description = "Path to the JSON configuration file of metadata transformers.") + private String transformerConfigFile; + } } diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java index 42876fba9..76a8197f4 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java @@ -2,6 +2,7 @@ import org.opensearch.migrations.MigrateOrEvaluateArgs; import org.opensearch.migrations.MigrationMode; +import org.opensearch.migrations.bulkload.transformers.CompositeTransformer; import org.opensearch.migrations.metadata.tracing.RootMetadataMigrationContext; import com.beust.jcommander.ParameterException; @@ -24,7 +25,10 @@ public EvaluateResult execute(RootMetadataMigrationContext context) { var clusters = createClusters(); evaluateResult.clusters(clusters); - var transformer = selectTransformer(clusters); + var transformer = new CompositeTransformer( + getCustomTransformer(), + selectTransformer(clusters) + ); var items = migrateAllItems(migrationMode, clusters, transformer, context); evaluateResult.items(items); diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java index 00edd304f..0f4e4335a 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java @@ -2,6 +2,7 @@ import org.opensearch.migrations.MigrateOrEvaluateArgs; import org.opensearch.migrations.MigrationMode; +import org.opensearch.migrations.bulkload.transformers.CompositeTransformer; import org.opensearch.migrations.metadata.tracing.RootMetadataMigrationContext; import com.beust.jcommander.ParameterException; @@ -24,7 +25,10 @@ public MigrateResult execute(RootMetadataMigrationContext context) { var clusters = createClusters(); migrateResult.clusters(clusters); - var transformer = selectTransformer(clusters); + var transformer = new CompositeTransformer( + getCustomTransformer(), + selectTransformer(clusters) + ); var items = migrateAllItems(migrationMode, clusters, transformer, context); migrateResult.items(items); diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java index b9871e08b..ecc363ccc 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java @@ -7,6 +7,7 @@ import org.opensearch.migrations.MigrationMode; import org.opensearch.migrations.bulkload.transformers.TransformFunctions; import org.opensearch.migrations.bulkload.transformers.Transformer; +import org.opensearch.migrations.bulkload.transformers.TransformerToIJsonTransformerAdapter; import org.opensearch.migrations.bulkload.worker.IndexMetadataResults; import org.opensearch.migrations.bulkload.worker.IndexRunner; import org.opensearch.migrations.bulkload.worker.MetadataRunner; @@ -17,12 +18,19 @@ import org.opensearch.migrations.metadata.CreationResult; import org.opensearch.migrations.metadata.GlobalMetadataCreatorResults; import org.opensearch.migrations.metadata.tracing.RootMetadataMigrationContext; +import org.opensearch.migrations.transform.TransformationLoader; +import org.opensearch.migrations.transform.TransformerConfigUtils; import lombok.extern.slf4j.Slf4j; /** Shared functionality between migration and evaluation commands */ @Slf4j public abstract class MigratorEvaluatorBase { + public static final String NOOP_TRANSFORMATION_CONFIG = "[" + + " {" + + " \"NoopTransformerProvider\":\"\"" + + " }" + + "]"; static final int INVALID_PARAMETER_CODE = 999; static final int UNEXPECTED_FAILURE_CODE = 888; @@ -45,13 +53,27 @@ protected Clusters createClusters() { return clusters.build(); } + protected Transformer getCustomTransformer() { + var transformerConfig = TransformerConfigUtils.getTransformerConfig(arguments.metadataTransformationParams); + if (transformerConfig != null) { + log.atInfo().setMessage("Metadata Transformations config string: {}") + .addArgument(transformerConfig).log(); + } else { + log.atInfo().setMessage("Using Noop transformation config: {}") + .addArgument(NOOP_TRANSFORMATION_CONFIG).log(); + transformerConfig = NOOP_TRANSFORMATION_CONFIG; + } + var transformer = new TransformationLoader().getTransformerFactoryLoader(transformerConfig); + return new TransformerToIJsonTransformerAdapter(transformer); + } + protected Transformer selectTransformer(Clusters clusters) { var transformer = TransformFunctions.getTransformer( clusters.getSource().getVersion(), clusters.getTarget().getVersion(), arguments.minNumberOfReplicas ); - log.info("Selected transformer " + transformer.toString()); + log.atInfo().setMessage("Selected transformer: {}").addArgument(transformer).log(); return transformer; } diff --git a/MetadataMigration/src/main/resources/log4j2.properties b/MetadataMigration/src/main/resources/log4j2.properties index 670c04fb9..a55b999a2 100644 --- a/MetadataMigration/src/main/resources/log4j2.properties +++ b/MetadataMigration/src/main/resources/log4j2.properties @@ -2,8 +2,9 @@ status = WARN property.logsDir = ${env:SHARED_LOGS_DIR_PATH:-./logs} property.failedLoggerFileNamePrefix = ${logsDir}/${hostName}/failedRequests/failedRequests +property.metadataTuplesFileNamePrefix = ${logsDir}/${hostName}/metadataTuples/tuples -appenders = console, FailedRequests, MetadataRun +appenders = console, FailedRequests, MetadataRun, MetadataTuples appender.FailedRequests.type = RollingRandomAccessFile appender.FailedRequests.name = FailedRequests @@ -51,3 +52,20 @@ logger.MetadataMigration.level = info logger.MetadataMigration.additivity = false logger.MetadataMigration.appenderRef.stdout.ref = Console logger.MetadataMigration.appenderRef.MetadataRun.ref = MetadataRun + +appender.MetadataTuples.type = RollingRandomAccessFile +appender.MetadataTuples.name = MetadataTuples +appender.MetadataTuples.fileName = ${metadataTuplesFileNamePrefix}.log +appender.MetadataTuples.filePattern = ${metadataTuplesFileNamePrefix}-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log +appender.MetadataTuples.layout.type = PatternLayout +appender.MetadataTuples.layout.pattern = %m%n +appender.MetadataTuples.policies.type = Policies +appender.MetadataTuples.policies.size.type = SizeBasedTriggeringPolicy +appender.MetadataTuples.policies.size.size = 10 MB +appender.MetadataTuples.strategy.type = DefaultRolloverStrategy +appender.MetadataTuples.immediateFlush = false + +logger.OutputTransformationJsonLogger.name = OutputTransformationJsonLogger +logger.OutputTransformationJsonLogger.level = info +logger.OutputTransformationJsonLogger.additivity = false +logger.OutputTransformationJsonLogger.appenderRef.MetadataTuples.ref = MetadataTuples diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/ComponentTemplate.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/ComponentTemplate.java new file mode 100644 index 000000000..a019b871c --- /dev/null +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/ComponentTemplate.java @@ -0,0 +1,13 @@ +package org.opensearch.migrations.bulkload.models; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +@NoArgsConstructor(force = true, access = AccessLevel.PRIVATE) // For Jackson +public class ComponentTemplate extends MigrationItem { + public static final String TYPE = "component_template"; + public ComponentTemplate(final String name, final ObjectNode body) { + super(TYPE, name, body); + } +} diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/GlobalMetadata.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/GlobalMetadata.java index a8b65a2be..2d867274e 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/GlobalMetadata.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/GlobalMetadata.java @@ -9,6 +9,7 @@ import org.opensearch.migrations.bulkload.common.RfsException; import org.opensearch.migrations.bulkload.common.SnapshotRepo; +import com.fasterxml.jackson.core.JsonPointer; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -21,18 +22,34 @@ public interface GlobalMetadata { * See: https://github.com/elastic/elasticsearch/blob/v7.10.2/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java#L1622 * See: https://github.com/elastic/elasticsearch/blob/v6.8.23/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java#L1214 */ - public ObjectNode toObjectNode(); + ObjectNode toObjectNode(); - public ObjectNode getTemplates(); + JsonPointer getTemplatesPath(); - public ObjectNode getIndexTemplates(); + JsonPointer getIndexTemplatesPath(); - public ObjectNode getComponentTemplates(); + JsonPointer getComponentTemplatesPath(); + + default ObjectNode getTemplates() { + return getObjectNodeWithPath(getTemplatesPath()); + } + + default ObjectNode getIndexTemplates() { + return getObjectNodeWithPath(getIndexTemplatesPath()); + } + + default ObjectNode getComponentTemplates() { + return getObjectNodeWithPath(getComponentTemplatesPath()); + } + + default ObjectNode getObjectNodeWithPath(JsonPointer path) { + return toObjectNode().withObject(path, JsonNode.OverwriteMode.NULLS, false); + } /** * Defines the behavior required to read a snapshot's global metadata as JSON and convert it into a Data object */ - public static interface Factory { + interface Factory { private JsonNode getJsonNode( SnapshotRepo.Provider repoDataProvider, String snapshotName, @@ -73,22 +90,22 @@ default GlobalMetadata fromRepo(String snapshotName) { } // Version-specific implementation - public GlobalMetadata fromJsonNode(JsonNode root); + GlobalMetadata fromJsonNode(JsonNode root); // Version-specific implementation - public SmileFactory getSmileFactory(); + SmileFactory getSmileFactory(); // Get the underlying SnapshotRepo Provider - public SnapshotRepo.Provider getRepoDataProvider(); + SnapshotRepo.Provider getRepoDataProvider(); } - public static class CantFindSnapshotInRepo extends RfsException { + class CantFindSnapshotInRepo extends RfsException { public CantFindSnapshotInRepo(String snapshotName) { super("Can't find snapshot in repo: " + snapshotName); } } - public static class CantReadGlobalMetadataFromSnapshot extends RfsException { + class CantReadGlobalMetadataFromSnapshot extends RfsException { public CantReadGlobalMetadataFromSnapshot(String snapshotName, Throwable cause) { super("Can't read the global metadata from snapshot: " + snapshotName, cause); } diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/Index.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/Index.java new file mode 100644 index 000000000..cef345ad0 --- /dev/null +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/Index.java @@ -0,0 +1,13 @@ +package org.opensearch.migrations.bulkload.models; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +@NoArgsConstructor(force = true, access = AccessLevel.PRIVATE) // For Jackson +public class Index extends MigrationItem { + public final static String TYPE = "index"; + public Index(String name, ObjectNode body) { + super(TYPE, name, body); + } +} diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/IndexMetadata.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/IndexMetadata.java index dcc6d7d8c..f6a8ecabd 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/IndexMetadata.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/IndexMetadata.java @@ -10,31 +10,33 @@ import org.opensearch.migrations.bulkload.common.SnapshotRepo; import org.opensearch.migrations.transformation.entity.Index; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.smile.SmileFactory; import org.apache.lucene.codecs.CodecUtil; -public interface IndexMetadata extends Index { - +// All subclasses need to be annotated with this +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "type") +public abstract class IndexMetadata implements Index { /* * Defines the behavior expected of an object that will surface the metadata of an index stored in a snapshot * See: https://github.com/elastic/elasticsearch/blob/v7.10.2/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java#L1475 * See: https://github.com/elastic/elasticsearch/blob/v6.8.23/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java#L1284 */ - public JsonNode getAliases(); + public abstract JsonNode getAliases(); - public String getId(); + public abstract String getId(); - public JsonNode getMappings(); + public abstract JsonNode getMappings(); - public String getName(); + public abstract String getName(); - public int getNumberOfShards(); + public abstract int getNumberOfShards(); - public JsonNode getSettings(); + public abstract JsonNode getSettings(); - public IndexMetadata deepCopy(); + public abstract IndexMetadata deepCopy(); /** * Defines the behavior required to read a snapshot's index metadata as JSON and convert it into a Data object @@ -71,15 +73,15 @@ default IndexMetadata fromRepo(String snapshotName, String indexName) { } // Version-specific implementation - public IndexMetadata fromJsonNode(JsonNode root, String indexId, String indexName); + IndexMetadata fromJsonNode(JsonNode root, String indexId, String indexName); // Version-specific implementation - public SmileFactory getSmileFactory(); + SmileFactory getSmileFactory(); // Version-specific implementation - public String getIndexFileId(String snapshotName, String indexName); + String getIndexFileId(String snapshotName, String indexName); // Get the underlying SnapshotRepo Provider - public SnapshotRepo.Provider getRepoDataProvider(); + SnapshotRepo.Provider getRepoDataProvider(); } } diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/IndexTemplate.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/IndexTemplate.java new file mode 100644 index 000000000..792e9d53e --- /dev/null +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/IndexTemplate.java @@ -0,0 +1,13 @@ +package org.opensearch.migrations.bulkload.models; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +@NoArgsConstructor(force = true, access = AccessLevel.PRIVATE) // For Jackson +public class IndexTemplate extends MigrationItem { + public static final String TYPE = "index_template"; + public IndexTemplate(final String name, final ObjectNode body) { + super(TYPE, name, body); + } +} diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/LegacyTemplate.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/LegacyTemplate.java new file mode 100644 index 000000000..7edfa763a --- /dev/null +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/LegacyTemplate.java @@ -0,0 +1,13 @@ +package org.opensearch.migrations.bulkload.models; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +@NoArgsConstructor(force = true, access = AccessLevel.PRIVATE) // For Jackson +public class LegacyTemplate extends MigrationItem { + public final static String TYPE = "template"; + public LegacyTemplate(final String name, final ObjectNode body) { + super(TYPE, name, body); + } +} diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/MigrationItem.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/MigrationItem.java new file mode 100644 index 000000000..dccd2cea1 --- /dev/null +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/MigrationItem.java @@ -0,0 +1,27 @@ +package org.opensearch.migrations.bulkload.models; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +@NoArgsConstructor(force = true, access = AccessLevel.PROTECTED) // For Jackson +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = Index.class, name = Index.TYPE), + @JsonSubTypes.Type(value = LegacyTemplate.class, name = LegacyTemplate.TYPE), + @JsonSubTypes.Type(value = IndexTemplate.class, name = IndexTemplate.TYPE), + @JsonSubTypes.Type(value = ComponentTemplate.class, name = ComponentTemplate.TYPE) +}) +public abstract class MigrationItem { + public final String type; + public final String name; + public final ObjectNode body; + + public MigrationItem(final String type, final String name, final ObjectNode body) { + this.type = type; + this.name = name; + this.body = body; + } +} diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/CompositeTransformer.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/CompositeTransformer.java new file mode 100644 index 000000000..075c5204b --- /dev/null +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/CompositeTransformer.java @@ -0,0 +1,28 @@ +package org.opensearch.migrations.bulkload.transformers; + +import org.opensearch.migrations.bulkload.models.GlobalMetadata; +import org.opensearch.migrations.bulkload.models.IndexMetadata; + +public class CompositeTransformer implements Transformer { + private final Transformer[] transformers; + + public CompositeTransformer(Transformer... transformers) { + this.transformers = transformers; + } + + @Override + public GlobalMetadata transformGlobalMetadata(GlobalMetadata globalData) { + for (Transformer transformer : transformers) { + globalData = transformer.transformGlobalMetadata(globalData); + } + return globalData; + } + + @Override + public IndexMetadata transformIndexMetadata(IndexMetadata indexData) { + for (Transformer transformer : transformers) { + indexData = transformer.transformIndexMetadata(indexData); + } + return indexData; + } +} diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformerToIJsonTransformerAdapter.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformerToIJsonTransformerAdapter.java new file mode 100644 index 000000000..eaf08cada --- /dev/null +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformerToIJsonTransformerAdapter.java @@ -0,0 +1,165 @@ +package org.opensearch.migrations.bulkload.transformers; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.opensearch.migrations.bulkload.models.ComponentTemplate; +import org.opensearch.migrations.bulkload.models.GlobalMetadata; +import org.opensearch.migrations.bulkload.models.IndexMetadata; +import org.opensearch.migrations.bulkload.models.IndexTemplate; +import org.opensearch.migrations.bulkload.models.LegacyTemplate; +import org.opensearch.migrations.bulkload.models.MigrationItem; +import org.opensearch.migrations.transform.IJsonTransformer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Lombok; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +@Slf4j +public class TransformerToIJsonTransformerAdapter implements Transformer { + public static final String OUTPUT_TRANSFORMATION_JSON_LOGGER = "OutputTransformationJsonLogger"; + private final static ObjectMapper MAPPER = new ObjectMapper(); + private final IJsonTransformer transformer; + private final Logger transformerLogger; + + public TransformerToIJsonTransformerAdapter(IJsonTransformer transformer, Logger transformerLogger) { + this.transformer = transformer; + this.transformerLogger = transformerLogger != null ? transformerLogger : LoggerFactory.getLogger(OUTPUT_TRANSFORMATION_JSON_LOGGER); + } + + public TransformerToIJsonTransformerAdapter(IJsonTransformer transformer) { + this(transformer, LoggerFactory.getLogger(OUTPUT_TRANSFORMATION_JSON_LOGGER)); + } + + private void logTransformation(Map before, Map after) { + if (transformerLogger.isInfoEnabled()) { + try { + var transformationTuple = toTransformationMap(before, after); + var tupleString = MAPPER.writeValueAsString(transformationTuple); + transformerLogger.atInfo().setMessage("{}").addArgument(tupleString).log(); + } catch (Exception e) { + log.atError().setCause(e).setMessage("Exception converting tuple to string").log(); + transformerLogger.atInfo().setMessage("{ \"error\":\"{}\" }").addArgument(e::getMessage).log(); + throw Lombok.sneakyThrow(e); + } + } + } + + private Map toTransformationMap(Map before, Map after) { + var transformationMap = new LinkedHashMap(); + transformationMap.put("before", before); + transformationMap.put("after", after); + return transformationMap; + } + + + @SuppressWarnings("unchecked") + private static Map objectNodeToMap(Object node) { + return (Map) MAPPER.convertValue(node, Map.class); + } + + @SneakyThrows + private static String printMap(Map map) { + return MAPPER.writeValueAsString(map); + } + + @SuppressWarnings("unchecked") + private MigrationItem transformMigrationItem(MigrationItem migrationItem) { + // Keep untouched original for logging + final Map originalMap = MAPPER.convertValue(migrationItem, Map.class); + var transformedMigrationItem = transformer.transformJson(MAPPER.convertValue(migrationItem, Map.class)); + logTransformation(originalMap, transformedMigrationItem); + return MAPPER.convertValue(transformedMigrationItem, MigrationItem.class); + } + + void updateTemplates(Collection transformedItems, ObjectNode itemsRoot) { + itemsRoot.removeAll(); + transformedItems.forEach(item -> + { + log.atInfo().setMessage("Setting new item of type {}, name {}, body {}") + .addArgument(item.type) + .addArgument(item.name) + .addArgument(item.body) + .log(); + itemsRoot.set(item.name, item.body); + } + ); + } + + @Override + public GlobalMetadata transformGlobalMetadata(GlobalMetadata globalData) { + var inputJson = objectNodeToMap(globalData.toObjectNode()); + log.atInfo().setMessage("BeforeJsonGlobal: {}").addArgument(() -> printMap(inputJson)).log(); + var afterJson = transformer.transformJson(inputJson); + log.atInfo().setMessage("AfterJsonGlobal: {}").addArgument(() -> printMap(afterJson)).log(); + + + final List legacyTemplates = new ArrayList<>(); + globalData.getTemplates().fields().forEachRemaining( + entry -> legacyTemplates.add(new LegacyTemplate(entry.getKey(), (ObjectNode) entry.getValue())) + ); + final List indexTemplates = new ArrayList<>(); + globalData.getIndexTemplates().fields().forEachRemaining( + entry -> indexTemplates.add(new IndexTemplate(entry.getKey(), (ObjectNode) entry.getValue())) + ); + final List componentTemplates = new ArrayList<>(); + globalData.getComponentTemplates().fields().forEachRemaining( + entry -> componentTemplates.add(new ComponentTemplate(entry.getKey(), (ObjectNode) entry.getValue())) + ); + + var transformedTemplates = Stream.concat(Stream.concat( + legacyTemplates.stream(), + indexTemplates.stream()), + componentTemplates.stream() + ) + .map(this::transformMigrationItem).collect(Collectors.toList()); + + var transformedLegacy = transformedTemplates.stream().filter( + item -> item instanceof LegacyTemplate + ) + .map(item -> (LegacyTemplate) item) + .collect(Collectors.toList()); + + var transformedIndex = transformedTemplates.stream().filter( + item -> item instanceof IndexTemplate + ) + .map(item -> (IndexTemplate) item) + .collect(Collectors.toList()); + + var transformedComponent = transformedTemplates.stream().filter( + item -> item instanceof ComponentTemplate + ) + .map(item -> (ComponentTemplate) item) + .collect(Collectors.toList()); + + assert transformedLegacy.size() + transformedIndex.size() + transformedComponent.size() == transformedTemplates.size(); + + updateTemplates(transformedLegacy, globalData.getTemplates()); + updateTemplates(transformedIndex, globalData.getIndexTemplates()); + updateTemplates(transformedComponent, globalData.getComponentTemplates()); + + log.atInfo().setMessage("GlobalOutput: {}").addArgument(() -> printMap(objectNodeToMap(globalData.toObjectNode()))).log(); + return globalData; + } + + @SuppressWarnings("unchecked") + @Override + public IndexMetadata transformIndexMetadata(IndexMetadata indexData) { + final Map originalInput = MAPPER.convertValue(indexData, Map.class); + final Map inputJson = MAPPER.convertValue(indexData, Map.class); + var afterJson = transformer.transformJson(inputJson); + logTransformation(originalInput, afterJson); + return MAPPER.convertValue(inputJson, IndexMetadata.class); + } + +} diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_6_8/GlobalMetadataData_ES_6_8.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_6_8/GlobalMetadataData_ES_6_8.java index 42756a474..e351243ed 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_6_8/GlobalMetadataData_ES_6_8.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_6_8/GlobalMetadataData_ES_6_8.java @@ -2,6 +2,7 @@ import org.opensearch.migrations.bulkload.models.GlobalMetadata; +import com.fasterxml.jackson.core.JsonPointer; import com.fasterxml.jackson.databind.node.ObjectNode; public class GlobalMetadataData_ES_6_8 implements GlobalMetadata { @@ -16,17 +17,18 @@ public ObjectNode toObjectNode() { return root; } - public ObjectNode getTemplates() { - return (ObjectNode) root.get("templates"); + @Override + public JsonPointer getTemplatesPath() { + return JsonPointer.compile("/templates"); } @Override - public ObjectNode getIndexTemplates() { - return null; + public JsonPointer getIndexTemplatesPath() { + return JsonPointer.compile("/index_template"); } @Override - public ObjectNode getComponentTemplates() { - return null; + public JsonPointer getComponentTemplatesPath() { + return JsonPointer.compile("/component_template"); } } diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_6_8/IndexMetadataData_ES_6_8.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_6_8/IndexMetadataData_ES_6_8.java index 0effc88ce..09b9ea586 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_6_8/IndexMetadataData_ES_6_8.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_6_8/IndexMetadataData_ES_6_8.java @@ -3,18 +3,28 @@ import org.opensearch.migrations.bulkload.models.IndexMetadata; import org.opensearch.migrations.bulkload.transformers.TransformFunctions; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Getter; +import lombok.NoArgsConstructor; -public class IndexMetadataData_ES_6_8 implements IndexMetadata { +@NoArgsConstructor(force = true) // For Jackson +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "type") +public class IndexMetadataData_ES_6_8 extends IndexMetadata { @Getter + @JsonProperty("body") private final ObjectNode rawJson; private ObjectNode mappings; private ObjectNode settings; @Getter + @JsonProperty("id") private final String id; @Getter + @JsonProperty("name") private final String name; public IndexMetadataData_ES_6_8(ObjectNode root, String indexId, String indexName) { diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_7_10/GlobalMetadataData_ES_7_10.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_7_10/GlobalMetadataData_ES_7_10.java index 1b0f41a54..67565b7e8 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_7_10/GlobalMetadataData_ES_7_10.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_7_10/GlobalMetadataData_ES_7_10.java @@ -1,11 +1,12 @@ package org.opensearch.migrations.bulkload.version_es_7_10; -import java.util.Optional; import org.opensearch.migrations.bulkload.models.GlobalMetadata; +import com.fasterxml.jackson.core.JsonPointer; import com.fasterxml.jackson.databind.node.ObjectNode; + public class GlobalMetadataData_ES_7_10 implements GlobalMetadata { private final ObjectNode root; @@ -18,19 +19,18 @@ public ObjectNode toObjectNode() { return root; } - public ObjectNode getTemplates() { - return Optional.ofNullable(root) - .map(node -> node.get("templates")) - .filter(ObjectNode.class::isInstance) - .map(ObjectNode.class::cast) - .orElse(null); + @Override + public JsonPointer getTemplatesPath() { + return JsonPointer.compile("/templates"); } - public ObjectNode getIndexTemplates() { - return (ObjectNode) root.get("index_template").get("index_template"); + @Override + public JsonPointer getIndexTemplatesPath() { + return JsonPointer.compile("/index_template/index_template"); } - public ObjectNode getComponentTemplates() { - return (ObjectNode) root.get("component_template").get("component_template"); + @Override + public JsonPointer getComponentTemplatesPath() { + return JsonPointer.compile("/component_template/component_template"); } } diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_7_10/IndexMetadataData_ES_7_10.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_7_10/IndexMetadataData_ES_7_10.java index 2377704f6..549422a06 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_7_10/IndexMetadataData_ES_7_10.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_es_7_10/IndexMetadataData_ES_7_10.java @@ -3,17 +3,27 @@ import org.opensearch.migrations.bulkload.models.IndexMetadata; import org.opensearch.migrations.bulkload.transformers.TransformFunctions; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Getter; +import lombok.NoArgsConstructor; -public class IndexMetadataData_ES_7_10 implements IndexMetadata { +@NoArgsConstructor(force = true) // For Jackson +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "type") +public class IndexMetadataData_ES_7_10 extends IndexMetadata { @Getter + @JsonProperty("body") private final ObjectNode rawJson; private ObjectNode mappings; private ObjectNode settings; @Getter + @JsonProperty("id") private final String id; @Getter + @JsonProperty("name") private final String name; public IndexMetadataData_ES_7_10(ObjectNode root, String indexId, String indexName) { diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_os_2_11/GlobalMetadataData_OS_2_11.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_os_2_11/GlobalMetadataData_OS_2_11.java index a7c00b3f4..45bf8935f 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_os_2_11/GlobalMetadataData_OS_2_11.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_os_2_11/GlobalMetadataData_OS_2_11.java @@ -2,6 +2,7 @@ import org.opensearch.migrations.bulkload.models.GlobalMetadata; +import com.fasterxml.jackson.core.JsonPointer; import com.fasterxml.jackson.databind.node.ObjectNode; public class GlobalMetadataData_OS_2_11 implements GlobalMetadata { @@ -16,25 +17,18 @@ public ObjectNode toObjectNode() { return root; } - public ObjectNode getTemplates() { - return (ObjectNode) root.get("templates"); + @Override + public JsonPointer getTemplatesPath() { + return JsonPointer.compile("/templates"); } - public ObjectNode getIndexTemplates() { - String indexTemplateKey = "index_template"; - if (root.get(indexTemplateKey) != null) { - return (ObjectNode) root.get(indexTemplateKey).get(indexTemplateKey); - } else { - return null; - } + @Override + public JsonPointer getIndexTemplatesPath() { + return JsonPointer.compile("/index_template/index_template"); } - public ObjectNode getComponentTemplates() { - String componentTemplateKey = "component_template"; - if (root.get(componentTemplateKey) != null) { - return (ObjectNode) root.get(componentTemplateKey).get(componentTemplateKey); - } else { - return null; - } + @Override + public JsonPointer getComponentTemplatesPath() { + return JsonPointer.compile("/component_template/component_template"); } } diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_os_2_11/IndexMetadataData_OS_2_11.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_os_2_11/IndexMetadataData_OS_2_11.java index c621e3861..067b3b78e 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_os_2_11/IndexMetadataData_OS_2_11.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_os_2_11/IndexMetadataData_OS_2_11.java @@ -2,11 +2,21 @@ import org.opensearch.migrations.bulkload.models.IndexMetadata; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.NoArgsConstructor; -public class IndexMetadataData_OS_2_11 implements IndexMetadata { +@NoArgsConstructor(force = true) // For Jackson +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "type") +public class IndexMetadataData_OS_2_11 extends IndexMetadata { + @JsonProperty("body") private ObjectNode root; + @JsonProperty("id") private String indexId; + @JsonProperty("name") private String indexName; public IndexMetadataData_OS_2_11(ObjectNode root, String indexId, String indexName) { diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_universal/RemoteIndexMetadata.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_universal/RemoteIndexMetadata.java index 7f830c63a..70b229420 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_universal/RemoteIndexMetadata.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_universal/RemoteIndexMetadata.java @@ -2,55 +2,56 @@ import org.opensearch.migrations.bulkload.models.IndexMetadata; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; -@AllArgsConstructor -public class RemoteIndexMetadata implements IndexMetadata { +@NoArgsConstructor(force = true) // For Jackson +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE) +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "type") +@Getter +public class RemoteIndexMetadata extends IndexMetadata { + @JsonProperty("name") + private String name; + @JsonProperty("id") + private String id; + @JsonProperty("body") + private ObjectNode rawJson; - private String indexName; - private ObjectNode sourceData; - - @Override - public ObjectNode getRawJson() { - return sourceData; + RemoteIndexMetadata(String indexName, ObjectNode rawJson) { + this.name = indexName; + // ID is the same as name in remote metadata + this.id = indexName; + this.rawJson = rawJson; } @Override public JsonNode getAliases() { - return sourceData.get("aliases"); - } - - @Override - public String getId() { - // The ID is the name in this case - return getName(); + return rawJson.get("aliases"); } @Override public JsonNode getMappings() { - return sourceData.get("mappings"); - } - - @Override - public String getName() { - return indexName; + return rawJson.get("mappings"); } @Override public int getNumberOfShards() { - throw new UnsupportedOperationException("Unimplemented method 'getNumberOfShards'"); + return getSettings().get("index").get("number_of_shards").asInt(); } @Override public JsonNode getSettings() { - return sourceData.get("settings"); + return rawJson.get("settings"); } @Override public IndexMetadata deepCopy() { - return new RemoteIndexMetadata(indexName, sourceData.deepCopy()); + return new RemoteIndexMetadata(name, rawJson.deepCopy()); } } diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_universal/RemoteMetadata.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_universal/RemoteMetadata.java index 36aa46766..0b6247ab2 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_universal/RemoteMetadata.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_universal/RemoteMetadata.java @@ -1,9 +1,8 @@ package org.opensearch.migrations.bulkload.version_universal; -import java.util.Optional; - import org.opensearch.migrations.bulkload.models.GlobalMetadata; +import com.fasterxml.jackson.core.JsonPointer; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.AllArgsConstructor; @@ -18,30 +17,17 @@ public ObjectNode toObjectNode() { } @Override - public ObjectNode getTemplates() { - return Optional.ofNullable(sourceData) - .map(node -> node.get("templates")) - .map(node -> node.get("templates")) - .filter(ObjectNode.class::isInstance) - .map(ObjectNode.class::cast) - .orElse(null); + public JsonPointer getTemplatesPath() { + return JsonPointer.compile("/templates/templates"); } @Override - public ObjectNode getIndexTemplates() { - return Optional.ofNullable(sourceData) - .map(node -> node.get("index_template")) - .map(node -> node.get("index_template")) - .filter(ObjectNode.class::isInstance) - .map(ObjectNode.class::cast) - .orElse(null); } + public JsonPointer getIndexTemplatesPath() { + return JsonPointer.compile("/index_template/index_template"); + } @Override - public ObjectNode getComponentTemplates() { - return Optional.ofNullable(sourceData) - .map(node -> node.get("component_template")) - .map(node -> node.get("component_template")) - .filter(ObjectNode.class::isInstance) - .map(ObjectNode.class::cast) - .orElse(null); } + public JsonPointer getComponentTemplatesPath() { + return JsonPointer.compile("/component_template/component_template"); + } } diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/worker/IndexRunner.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/worker/IndexRunner.java index 2d287b94a..7592dbd80 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/worker/IndexRunner.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/worker/IndexRunner.java @@ -2,6 +2,7 @@ import java.util.List; import java.util.function.BiConsumer; +import java.util.function.Predicate; import org.opensearch.migrations.MigrationMode; import org.opensearch.migrations.bulkload.common.FilterScheme; @@ -35,16 +36,34 @@ public IndexMetadataResults migrateIndices(MigrationMode mode, ICreateIndexConte }; var results = IndexMetadataResults.builder(); + // log out filtered items + repoDataProvider.getIndicesInSnapshot(snapshotName) + .stream() + .filter(Predicate.not(FilterScheme.filterIndicesByAllowList(indexAllowlist, logger))) + .forEach(index -> { + var indexMetadata = metadataFactory.fromRepo(snapshotName, index.getName()); + log.atInfo().setMessage("{ \"before\": {},\n\"after\":{}}") + .addArgument(indexMetadata) + .addArgument("Removed due to index filter") + .log(); + }); + + repoDataProvider.getIndicesInSnapshot(snapshotName) .stream() .filter(FilterScheme.filterIndicesByAllowList(indexAllowlist, logger)) .forEach(index -> { var indexName = index.getName(); - var indexMetadata = metadataFactory.fromRepo(snapshotName, indexName); + var originalIndexMetadata = metadataFactory.fromRepo(snapshotName, indexName); CreationResult indexResult = null; + var indexMetadata = originalIndexMetadata.deepCopy(); try { indexMetadata = transformer.transformIndexMetadata(indexMetadata); + log.atInfo().setMessage("{ \"before\": {},\n\"after\":{}}") + .addArgument(originalIndexMetadata) + .addArgument(indexMetadata) + .log(); indexResult = indexCreator.create(indexMetadata, mode, context); } catch (Throwable t) { indexResult = CreationResult.builder() diff --git a/transformation/src/main/java/org/opensearch/migrations/transformation/JsonTransformerForDocumentTypeRemovalProvider.java b/transformation/src/main/java/org/opensearch/migrations/transformation/JsonTransformerForDocumentTypeRemovalProvider.java index ab2b852df..df8f9d083 100644 --- a/transformation/src/main/java/org/opensearch/migrations/transformation/JsonTransformerForDocumentTypeRemovalProvider.java +++ b/transformation/src/main/java/org/opensearch/migrations/transformation/JsonTransformerForDocumentTypeRemovalProvider.java @@ -18,7 +18,9 @@ private static class Transformer implements IJsonTransformer { @Override @SuppressWarnings("unchecked") public Map transformJson(Map incomingJson) { - ((Map) incomingJson.get("index")).remove("_type"); + if (incomingJson.containsKey("index")) { + ((Map) incomingJson.get("index")).remove("_type"); + } return incomingJson; } } From ad1ba91aaaa1278d3e09b6260fd8b2b21ad6f9b1 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 22 Nov 2024 10:04:25 -0600 Subject: [PATCH 2/5] Update metadata transformations based on PR feedback Signed-off-by: Andre Kurait --- .../migrations/MigrateOrEvaluateArgs.java | 2 +- .../migrations/commands/Evaluate.java | 5 +- .../migrations/commands/Migrate.java | 5 +- .../commands/MigratorEvaluatorBase.java | 11 +- .../src/main/resources/log4j2.properties | 75 ++--- .../migrations/CustomTransformationTest.java | 281 ++++++++++++++++++ .../bulkload/worker/IndexRunner.java | 17 +- .../migrations/metadata/CreationResult.java | 3 +- 8 files changed, 337 insertions(+), 62 deletions(-) create mode 100644 MetadataMigration/src/test/java/org/opensearch/migrations/CustomTransformationTest.java diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java b/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java index 4c51a1868..01653a428 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java @@ -56,7 +56,7 @@ public class MigrateOrEvaluateArgs { public Version sourceVersion = null; @ParametersDelegate - public MetadataTransformerParams metadataTransformationParams = new MetadataTransformerParams(); + public TransformerParams metadataTransformationParams = new MetadataTransformerParams(); @Getter public static class MetadataTransformerParams implements TransformerParams { diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java index 76a8197f4..b1c4724a1 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java @@ -25,10 +25,7 @@ public EvaluateResult execute(RootMetadataMigrationContext context) { var clusters = createClusters(); evaluateResult.clusters(clusters); - var transformer = new CompositeTransformer( - getCustomTransformer(), - selectTransformer(clusters) - ); + var transformer = selectTransformer(clusters); var items = migrateAllItems(migrationMode, clusters, transformer, context); evaluateResult.items(items); diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java index 0f4e4335a..76106c41d 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java @@ -25,10 +25,7 @@ public MigrateResult execute(RootMetadataMigrationContext context) { var clusters = createClusters(); migrateResult.clusters(clusters); - var transformer = new CompositeTransformer( - getCustomTransformer(), - selectTransformer(clusters) - ); + var transformer = selectTransformer(clusters); var items = migrateAllItems(migrationMode, clusters, transformer, context); migrateResult.items(items); diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java index ecc363ccc..8a2dd863f 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java @@ -5,6 +5,7 @@ import org.opensearch.migrations.MigrateOrEvaluateArgs; import org.opensearch.migrations.MigrationMode; +import org.opensearch.migrations.bulkload.transformers.CompositeTransformer; import org.opensearch.migrations.bulkload.transformers.TransformFunctions; import org.opensearch.migrations.bulkload.transformers.Transformer; import org.opensearch.migrations.bulkload.transformers.TransformerToIJsonTransformerAdapter; @@ -59,7 +60,7 @@ protected Transformer getCustomTransformer() { log.atInfo().setMessage("Metadata Transformations config string: {}") .addArgument(transformerConfig).log(); } else { - log.atInfo().setMessage("Using Noop transformation config: {}") + log.atInfo().setMessage("Using Noop custom transformation config: {}") .addArgument(NOOP_TRANSFORMATION_CONFIG).log(); transformerConfig = NOOP_TRANSFORMATION_CONFIG; } @@ -68,13 +69,15 @@ protected Transformer getCustomTransformer() { } protected Transformer selectTransformer(Clusters clusters) { - var transformer = TransformFunctions.getTransformer( + var versionTransformer = TransformFunctions.getTransformer( clusters.getSource().getVersion(), clusters.getTarget().getVersion(), arguments.minNumberOfReplicas ); - log.atInfo().setMessage("Selected transformer: {}").addArgument(transformer).log(); - return transformer; + var customTransformer = getCustomTransformer(); + var compositeTransformer = new CompositeTransformer(customTransformer, versionTransformer); + log.atInfo().setMessage("Selected transformer: {}").addArgument(compositeTransformer).log(); + return compositeTransformer; } protected Items migrateAllItems(MigrationMode migrationMode, Clusters clusters, Transformer transformer, RootMetadataMigrationContext context) { diff --git a/MetadataMigration/src/main/resources/log4j2.properties b/MetadataMigration/src/main/resources/log4j2.properties index a55b999a2..ed2037018 100644 --- a/MetadataMigration/src/main/resources/log4j2.properties +++ b/MetadataMigration/src/main/resources/log4j2.properties @@ -3,40 +3,10 @@ status = WARN property.logsDir = ${env:SHARED_LOGS_DIR_PATH:-./logs} property.failedLoggerFileNamePrefix = ${logsDir}/${hostName}/failedRequests/failedRequests property.metadataTuplesFileNamePrefix = ${logsDir}/${hostName}/metadataTuples/tuples - -appenders = console, FailedRequests, MetadataRun, MetadataTuples - -appender.FailedRequests.type = RollingRandomAccessFile -appender.FailedRequests.name = FailedRequests -appender.FailedRequests.fileName = ${failedLoggerFileNamePrefix}.log -appender.FailedRequests.filePattern = ${failedLoggerFileNamePrefix}-%d{yyyy-MM-dd-HH:mm}{UTC}-%i.log.gz -appender.FailedRequests.layout.type = PatternLayout -appender.FailedRequests.layout.pattern = %m%n -appender.FailedRequests.policies.type = Policies -appender.FailedRequests.policies.size.type = SizeBasedTriggeringPolicy -appender.FailedRequests.policies.size.size = 10 MB -appender.FailedRequests.strategy.type = DefaultRolloverStrategy -appender.FailedRequests.immediateFlush = false - -logger.FailedRequestsLogger.name = FailedRequestsLogger -logger.FailedRequestsLogger.level = info -logger.FailedRequestsLogger.additivity = false -logger.FailedRequestsLogger.appenderRef.FailedRequests.ref = FailedRequests - property.runTime = ${date:yyyy-MM-dd_HH-mm-ss} -property.metadataRunLoggerFileNamePrefix = ${logsDir}/${hostName}/metadata/metadata_ - -appender.MetadataRun.type = File -appender.MetadataRun.name = MetadataRun -appender.MetadataRun.fileName = ${metadataRunLoggerFileNamePrefix}${runTime}.log -appender.MetadataRun.layout.type = PatternLayout -appender.MetadataRun.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS}{UTC} %p %c{1.} [%t] %m%n -appender.MetadataRun.immediateFlush = true +property.metadataRunLoggerFileNamePrefix = ${logsDir}/${hostName}/metadata/metadata -logger.MetadataLogger.name = MetadataLogger -logger.MetadataLogger.level = debug -logger.MetadataLogger.additivity = false -logger.MetadataLogger.appenderRef.MetadataRun.ref = MetadataRun +appenders = console, MetadataTuples, FailedRequests, MetadataRun appender.console.type = Console appender.console.name = Console @@ -44,24 +14,26 @@ appender.console.target = SYSTEM_OUT appender.console.layout.type = PatternLayout appender.console.layout.pattern = %m%n -rootLogger.level = info +rootLogger.level = ERROR rootLogger.appenderRef.console.ref = MetadataRun +# Metadata Migration logger.MetadataMigration.name = org.opensearch.migrations.MetadataMigration logger.MetadataMigration.level = info logger.MetadataMigration.additivity = false logger.MetadataMigration.appenderRef.stdout.ref = Console logger.MetadataMigration.appenderRef.MetadataRun.ref = MetadataRun +# Metadata Tuples appender.MetadataTuples.type = RollingRandomAccessFile appender.MetadataTuples.name = MetadataTuples appender.MetadataTuples.fileName = ${metadataTuplesFileNamePrefix}.log -appender.MetadataTuples.filePattern = ${metadataTuplesFileNamePrefix}-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log +appender.MetadataTuples.filePattern = ${metadataTuplesFileNamePrefix}_${runTime}-%i.log appender.MetadataTuples.layout.type = PatternLayout appender.MetadataTuples.layout.pattern = %m%n appender.MetadataTuples.policies.type = Policies -appender.MetadataTuples.policies.size.type = SizeBasedTriggeringPolicy -appender.MetadataTuples.policies.size.size = 10 MB +appender.MetadataTuples.policies.startup.type = OnStartupTriggeringPolicy +appender.MetadataTuples.policies.startup.minSize = 0 appender.MetadataTuples.strategy.type = DefaultRolloverStrategy appender.MetadataTuples.immediateFlush = false @@ -69,3 +41,34 @@ logger.OutputTransformationJsonLogger.name = OutputTransformationJsonLogger logger.OutputTransformationJsonLogger.level = info logger.OutputTransformationJsonLogger.additivity = false logger.OutputTransformationJsonLogger.appenderRef.MetadataTuples.ref = MetadataTuples + +# MetadataRun Logs +appender.MetadataRun.type = File +appender.MetadataRun.name = MetadataRun +appender.MetadataRun.fileName = ${metadataRunLoggerFileNamePrefix}${runTime}-%i.log +appender.MetadataRun.layout.type = PatternLayout +appender.MetadataRun.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS}{UTC} %p %c{1.} [%t] %m%n +appender.MetadataRun.immediateFlush = false + +logger.MetadataLogger.name = MetadataLogger +logger.MetadataLogger.level = debug +logger.MetadataLogger.additivity = false +logger.MetadataLogger.appenderRef.MetadataRun.ref = MetadataRun + +# Failed Requestss +appender.FailedRequests.type = RollingRandomAccessFile +appender.FailedRequests.name = FailedRequests +appender.FailedRequests.fileName = ${failedLoggerFileNamePrefix}.log +appender.FailedRequests.filePattern = ${failedLoggerFileNamePrefix}-%d{yyyy-MM-dd-HH:mm}{UTC}-%i.log.gz +appender.FailedRequests.layout.type = PatternLayout +appender.FailedRequests.layout.pattern = %m%n +appender.FailedRequests.policies.type = Policies +appender.FailedRequests.policies.size.type = SizeBasedTriggeringPolicy +appender.FailedRequests.policies.size.size = 10 MB +appender.FailedRequests.strategy.type = DefaultRolloverStrategy +appender.FailedRequests.immediateFlush = false + +logger.FailedRequestsLogger.name = FailedRequestsLogger +logger.FailedRequestsLogger.level = info +logger.FailedRequestsLogger.additivity = false +logger.FailedRequestsLogger.appenderRef.FailedRequests.ref = FailedRequests diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/CustomTransformationTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/CustomTransformationTest.java new file mode 100644 index 000000000..e3c90d22d --- /dev/null +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/CustomTransformationTest.java @@ -0,0 +1,281 @@ +package org.opensearch.migrations; + +import java.io.File; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Stream; + +import org.opensearch.migrations.bulkload.SupportedClusters; +import org.opensearch.migrations.bulkload.common.FileSystemSnapshotCreator; +import org.opensearch.migrations.bulkload.common.OpenSearchClient; +import org.opensearch.migrations.bulkload.common.http.ConnectionContextTestParams; +import org.opensearch.migrations.bulkload.framework.SearchClusterContainer; +import org.opensearch.migrations.bulkload.http.ClusterOperations; +import org.opensearch.migrations.bulkload.models.DataFilterArgs; +import org.opensearch.migrations.bulkload.worker.SnapshotRunner; +import org.opensearch.migrations.commands.MigrationItemResult; +import org.opensearch.migrations.metadata.tracing.MetadataMigrationTestContext; +import org.opensearch.migrations.snapshot.creation.tracing.SnapshotTestContext; +import org.opensearch.migrations.transform.TransformerParams; + +import lombok.Builder; +import lombok.Data; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Test class to verify custom transformations during metadata migrations. + */ +@Tag("isolatedTest") +@Slf4j +class CustomTransformationTest { + + @TempDir + private File localDirectory; + + private static Stream scenarios() { + // Define scenarios with different source and target cluster versions + return SupportedClusters.sources().stream() + .flatMap(sourceCluster -> + SupportedClusters.targets().stream() + .map(targetCluster -> Arguments.of(sourceCluster, targetCluster)) + ); + } + + @ParameterizedTest(name = "Custom Transformation From {0} to {1}") + @MethodSource(value = "scenarios") + void customTransformationMetadataMigration( + SearchClusterContainer.ContainerVersion sourceVersion, + SearchClusterContainer.ContainerVersion targetVersion) { + try ( + final var sourceCluster = new SearchClusterContainer(sourceVersion); + final var targetCluster = new SearchClusterContainer(targetVersion) + ) { + performCustomTransformationTest(sourceCluster, targetCluster); + } + } + + @SneakyThrows + private void performCustomTransformationTest( + final SearchClusterContainer sourceCluster, + final SearchClusterContainer targetCluster + ) { + // Start both source and target clusters asynchronously + CompletableFuture.allOf( + CompletableFuture.runAsync(sourceCluster::start), + CompletableFuture.runAsync(targetCluster::start) + ).join(); + + var sourceOperations = new ClusterOperations(sourceCluster.getUrl()); + var targetOperations = new ClusterOperations(targetCluster.getUrl()); + + // Test data + var originalIndexName = "test_index"; + var transformedIndexName = "transformed_index"; + var documentId = "1"; + var documentContent = "{\"field\":\"value\"}"; + + // Create index and add a document on the source cluster + sourceOperations.createIndex(originalIndexName); + sourceOperations.createDocument(originalIndexName, documentId, documentContent); + + // Create legacy template + var legacyTemplateName = "legacy_template"; + var legacyTemplatePattern = "legacy_*"; + sourceOperations.createLegacyTemplate(legacyTemplateName, legacyTemplatePattern); + + // Create index template + var indexTemplateName = "index_template"; + var indexTemplatePattern = "index*"; + + // Create component template + var componentTemplateName = "component_template"; + var componentTemplateMode = "mode_value"; // Replace with actual mode if applicable + boolean newComponentCompatible = sourceCluster.getContainerVersion().getVersion().getMajor() >= 7; + if (newComponentCompatible) { + sourceOperations.createIndexTemplate(indexTemplateName, "dummy", indexTemplatePattern); + + var componentTemplateAdditionalParam = "additional_param"; // Replace with actual param if applicable + sourceOperations.createComponentTemplate(componentTemplateName, indexTemplateName, componentTemplateAdditionalParam, "index*"); + } + + // Create index that matches the templates + var legacyIndexName = "legacy_index"; + var indexIndexName = "index_index"; + sourceOperations.createIndex(legacyIndexName); + sourceOperations.createIndex(indexIndexName); + + // Define custom transformations for index, legacy, and component templates + String customTransformationJson = "[\n" + + " {\n" + + " \"JsonConditionalTransformerProvider\": [\n" + + " {\"JsonJMESPathPredicateProvider\": { \"script\": \"name == 'test_index'\"}},\n" + + " [\n" + + " {\"JsonJoltTransformerProvider\": { \n" + + " \"script\": {\n" + + " \"operation\": \"modify-overwrite-beta\",\n" + + " \"spec\": {\n" + + " \"name\": \"transformed_index\"\n" + + " }\n" + + " } \n" + + " }}\n" + + " ]\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"JsonConditionalTransformerProvider\": [\n" + + " {\"JsonJMESPathPredicateProvider\": { \"script\": \"type == 'template' && name == 'legacy_template'\"}},\n" + + " [\n" + + " {\"JsonJoltTransformerProvider\": { \n" + + " \"script\": {\n" + + " \"operation\": \"modify-overwrite-beta\",\n" + + " \"spec\": {\n" + + " \"name\": \"transformed_legacy_template\"\n" + + " }\n" + + " } \n" + + " }}\n" + + " ]\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"JsonConditionalTransformerProvider\": [\n" + + " {\"JsonJMESPathPredicateProvider\": { \"script\": \"type == 'index_template' && name == 'index_template'\"}},\n" + + " [\n" + + " {\"JsonJoltTransformerProvider\": { \n" + + " \"script\": {\n" + + " \"operation\": \"modify-overwrite-beta\",\n" + + " \"spec\": {\n" + + " \"name\": \"transformed_index_template\",\n" + + " \"body\": {\n" + + " \"composed_of\": {\n" + + " \"[0]\": \"transformed_component_template\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }}\n" + + " ]\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"JsonConditionalTransformerProvider\": [\n" + + " {\"JsonJMESPathPredicateProvider\": { \"script\": \"type == 'component_template' && name == 'component_template'\"}},\n" + + " [\n" + + " {\"JsonJoltTransformerProvider\": { \n" + + " \"script\": {\n" + + " \"operation\": \"modify-overwrite-beta\",\n" + + " \"spec\": {\n" + + " \"name\": \"transformed_component_template\"\n" + + " }\n" + + " } \n" + + " }}\n" + + " ]\n" + + " ]\n" + + " }\n" + + "]"; + + var arguments = new MigrateOrEvaluateArgs(); + + // Use SnapshotImage as the transfer medium + var snapshotName = "custom_transformation_snap"; + var snapshotContext = SnapshotTestContext.factory().noOtelTracking(); + var sourceClient = new OpenSearchClient(ConnectionContextTestParams.builder() + .host(sourceCluster.getUrl()) + .insecure(true) + .build() + .toConnectionContext()); + var snapshotCreator = new FileSystemSnapshotCreator( + snapshotName, + sourceClient, + SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, + List.of(), + snapshotContext.createSnapshotCreateContext() + ); + SnapshotRunner.runAndWaitForCompletion(snapshotCreator); + sourceCluster.copySnapshotData(localDirectory.toString()); + arguments.fileSystemRepoPath = localDirectory.getAbsolutePath(); + arguments.snapshotName = snapshotName; + arguments.sourceVersion = sourceCluster.getContainerVersion().getVersion(); + + arguments.targetArgs.host = targetCluster.getUrl(); + + // Set up data filters to include only the test index and templates + var dataFilterArgs = new DataFilterArgs(); + dataFilterArgs.indexAllowlist = List.of(originalIndexName, legacyIndexName, indexIndexName, transformedIndexName); + dataFilterArgs.indexTemplateAllowlist = List.of(indexTemplateName, legacyTemplateName, "transformed_legacy_template", "transformed_index_template"); + dataFilterArgs.componentTemplateAllowlist = List.of(componentTemplateName, "transformed_component_template"); + arguments.dataFilterArgs = dataFilterArgs; + + // Specify the custom transformer configuration + arguments.metadataTransformationParams = TestTransformationParams.builder() + .transformerConfig(customTransformationJson) + .build(); + + // Execute the migration with the custom transformation + var metadataContext = MetadataMigrationTestContext.factory().noOtelTracking(); + var metadata = new MetadataMigration(); + + MigrationItemResult result = metadata.migrate(arguments).execute(metadataContext); + + // Verify the migration result + log.info(result.asCliOutput()); + assertThat(result.getExitCode(), equalTo(0)); + + // Verify that the transformed index exists on the target cluster + var res = targetOperations.get("/" + transformedIndexName); + assertThat(res.getKey(), equalTo(200)); + assertThat(res.getValue(), containsString(transformedIndexName)); + + // Verify that the original index does not exist on the target cluster + res = targetOperations.get("/" + originalIndexName); + assertThat(res.getKey(), equalTo(404)); + + // Verify that the transformed legacy template exists on the target cluster + res = targetOperations.get("/_template/transformed_legacy_template"); + assertThat(res.getKey(), equalTo(200)); + assertThat(res.getValue(), containsString("transformed_legacy_template")); + + // Verify that the original legacy template does not exist on the target cluster + res = targetOperations.get("/_template/" + legacyTemplateName); + assertThat(res.getKey(), equalTo(404)); + + if (newComponentCompatible) { + // Verify that the transformed index template exists on the target cluster + res = targetOperations.get("/_index_template/transformed_index_template"); + assertThat(res.getKey(), equalTo(200)); + assertThat(res.getValue(), containsString("transformed_index_template")); + + // Verify that the original index template does not exist on the target cluster + res = targetOperations.get("/_index_template/" + indexTemplateName); + assertThat(res.getKey(), equalTo(404)); + + // Verify that the transformed component template exists on the target cluster + res = targetOperations.get("/_component_template/transformed_component_template"); + assertThat(res.getKey(), equalTo(200)); + assertThat(res.getValue(), containsString("transformed_component_template")); + + // Verify that the original component template does not exist on the target cluster + res = targetOperations.get("/_component_template/" + componentTemplateName); + assertThat(res.getKey(), equalTo(404)); + } + } + + @Data + @Builder + private static class TestTransformationParams implements TransformerParams { + @Builder.Default + private String transformerConfigParameterArgPrefix = ""; + private String transformerConfigEncoded; + private String transformerConfig; + private String transformerConfigFile; + } +} diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/worker/IndexRunner.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/worker/IndexRunner.java index 7592dbd80..ddd9a0d85 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/worker/IndexRunner.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/worker/IndexRunner.java @@ -36,17 +36,14 @@ public IndexMetadataResults migrateIndices(MigrationMode mode, ICreateIndexConte }; var results = IndexMetadataResults.builder(); - // log out filtered items + // Set results for filtered items repoDataProvider.getIndicesInSnapshot(snapshotName) .stream() .filter(Predicate.not(FilterScheme.filterIndicesByAllowList(indexAllowlist, logger))) - .forEach(index -> { - var indexMetadata = metadataFactory.fromRepo(snapshotName, index.getName()); - log.atInfo().setMessage("{ \"before\": {},\n\"after\":{}}") - .addArgument(indexMetadata) - .addArgument("Removed due to index filter") - .log(); - }); + .forEach(index -> results.index(CreationResult.builder() + .name(index.getName()) + .failureType(CreationFailureType.SKIPPED_DUE_TO_FILTER) + .build())); repoDataProvider.getIndicesInSnapshot(snapshotName) @@ -60,10 +57,6 @@ public IndexMetadataResults migrateIndices(MigrationMode mode, ICreateIndexConte var indexMetadata = originalIndexMetadata.deepCopy(); try { indexMetadata = transformer.transformIndexMetadata(indexMetadata); - log.atInfo().setMessage("{ \"before\": {},\n\"after\":{}}") - .addArgument(originalIndexMetadata) - .addArgument(indexMetadata) - .log(); indexResult = indexCreator.create(indexMetadata, mode, context); } catch (Throwable t) { indexResult = CreationResult.builder() diff --git a/RFS/src/main/java/org/opensearch/migrations/metadata/CreationResult.java b/RFS/src/main/java/org/opensearch/migrations/metadata/CreationResult.java index 08e2ae2ed..243347d42 100644 --- a/RFS/src/main/java/org/opensearch/migrations/metadata/CreationResult.java +++ b/RFS/src/main/java/org/opensearch/migrations/metadata/CreationResult.java @@ -29,7 +29,8 @@ public boolean wasFatal() { public static enum CreationFailureType { ALREADY_EXISTS(false, "already exists"), UNABLE_TO_TRANSFORM_FAILURE(true, "failed to transform to the target version"), - TARGET_CLUSTER_FAILURE(true, "failed on target cluster"); + TARGET_CLUSTER_FAILURE(true, "failed on target cluster"), + SKIPPED_DUE_TO_FILTER(false, "skipped due to filter"); private final boolean fatal; private final String message; From be777d6eebaa4fb155bca62f57a262e8557fc2d1 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Mon, 25 Nov 2024 10:44:11 -0600 Subject: [PATCH 3/5] Fix spotless Signed-off-by: Andre Kurait --- .../main/java/org/opensearch/migrations/commands/Evaluate.java | 1 - .../main/java/org/opensearch/migrations/commands/Migrate.java | 1 - 2 files changed, 2 deletions(-) diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java index b1c4724a1..42876fba9 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java @@ -2,7 +2,6 @@ import org.opensearch.migrations.MigrateOrEvaluateArgs; import org.opensearch.migrations.MigrationMode; -import org.opensearch.migrations.bulkload.transformers.CompositeTransformer; import org.opensearch.migrations.metadata.tracing.RootMetadataMigrationContext; import com.beust.jcommander.ParameterException; diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java index 76106c41d..00edd304f 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java @@ -2,7 +2,6 @@ import org.opensearch.migrations.MigrateOrEvaluateArgs; import org.opensearch.migrations.MigrationMode; -import org.opensearch.migrations.bulkload.transformers.CompositeTransformer; import org.opensearch.migrations.metadata.tracing.RootMetadataMigrationContext; import com.beust.jcommander.ParameterException; From 769b107461807d3b43bae4a74f0706b23f3d9786 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Mon, 25 Nov 2024 11:46:40 -0600 Subject: [PATCH 4/5] Assert on successful migration items Signed-off-by: Andre Kurait --- .../org/opensearch/migrations/EndToEndTest.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java index f4a2c28ba..61e38d4f7 100644 --- a/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java @@ -206,10 +206,17 @@ private void verifyCommandResults( assertThat(result.getExitCode(), equalTo(0)); var migratedItems = result.getItems(); - assertThat(getNames(migratedItems.getIndexTemplates()), containsInAnyOrder(testData.indexTemplateName)); - assertThat(getNames(migratedItems.getComponentTemplates()), equalTo(templateType.equals(TemplateType.IndexAndComponent) ? List.of(testData.compoTemplateName) : List.of())); - assertThat(getNames(migratedItems.getIndexes()), containsInAnyOrder(testData.blogIndexName, testData.movieIndexName, testData.indexThatAlreadyExists)); - assertThat(getNames(migratedItems.getAliases()), containsInAnyOrder(testData.aliasInTemplate, testData.aliasName)); + assertThat(getNames(getSuccessfulResults(migratedItems.getIndexTemplates())), containsInAnyOrder(testData.indexTemplateName)); + assertThat(getNames(getSuccessfulResults(migratedItems.getComponentTemplates())), equalTo(templateType.equals(TemplateType.IndexAndComponent) ? List.of(testData.compoTemplateName) : List.of())); + assertThat(getNames(getSuccessfulResults(migratedItems.getIndexes())), containsInAnyOrder(testData.blogIndexName, testData.movieIndexName, testData.indexThatAlreadyExists)); + assertThat(getNames(getSuccessfulResults(migratedItems.getAliases())), containsInAnyOrder(testData.aliasInTemplate, testData.aliasName)); + + } + + private List getSuccessfulResults(List results) { + return results.stream() + .filter(CreationResult::wasSuccessful) + .collect(Collectors.toList()); } private List getNames(List items) { From 18e83bb95a2396b2703185a380e9932a89decfe4 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Mon, 25 Nov 2024 14:38:43 -0600 Subject: [PATCH 5/5] Correctly assert against ALREADY_EXISTS index Signed-off-by: Andre Kurait --- .../java/org/opensearch/migrations/EndToEndTest.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java index 61e38d4f7..183f24acf 100644 --- a/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java @@ -208,7 +208,8 @@ private void verifyCommandResults( var migratedItems = result.getItems(); assertThat(getNames(getSuccessfulResults(migratedItems.getIndexTemplates())), containsInAnyOrder(testData.indexTemplateName)); assertThat(getNames(getSuccessfulResults(migratedItems.getComponentTemplates())), equalTo(templateType.equals(TemplateType.IndexAndComponent) ? List.of(testData.compoTemplateName) : List.of())); - assertThat(getNames(getSuccessfulResults(migratedItems.getIndexes())), containsInAnyOrder(testData.blogIndexName, testData.movieIndexName, testData.indexThatAlreadyExists)); + assertThat(getNames(getSuccessfulResults(migratedItems.getIndexes())), containsInAnyOrder(testData.blogIndexName, testData.movieIndexName)); + assertThat(getNames(getFailedResultsByType(migratedItems.getIndexes(), CreationResult.CreationFailureType.ALREADY_EXISTS)), containsInAnyOrder(testData.indexThatAlreadyExists)); assertThat(getNames(getSuccessfulResults(migratedItems.getAliases())), containsInAnyOrder(testData.aliasInTemplate, testData.aliasName)); } @@ -219,6 +220,12 @@ private List getSuccessfulResults(List results) .collect(Collectors.toList()); } + private List getFailedResultsByType(List results, CreationResult.CreationFailureType failureType) { + return results.stream() + .filter(r -> failureType.equals(r.getFailureType())) + .collect(Collectors.toList()); + } + private List getNames(List items) { return items.stream().map(r -> r.getName()).collect(Collectors.toList()); }